16 #include "kmp_error.h" 20 #include "kmp_stats.h" 23 #include "ompt-specific.h" 26 #define MAX_MESSAGE 512 42 if ((env = getenv(
"KMP_INITIAL_THREAD_BIND")) != NULL &&
43 __kmp_str_match_true(env)) {
44 __kmp_middle_initialize();
45 KC_TRACE(10, (
"__kmpc_begin: middle initialization called\n"));
46 }
else if (__kmp_ignore_mppbeg() == FALSE) {
48 __kmp_internal_begin();
49 KC_TRACE(10, (
"__kmpc_begin: called\n"));
67 if (__kmp_ignore_mppend() == FALSE) {
68 KC_TRACE(10, (
"__kmpc_end: called\n"));
69 KA_TRACE(30, (
"__kmpc_end\n"));
71 __kmp_internal_end_thread(-1);
94 kmp_int32 gtid = __kmp_entry_gtid();
96 KC_TRACE(10, (
"__kmpc_global_thread_num: T#%d\n", gtid));
117 (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
119 return TCR_4(__kmp_all_nth);
129 KC_TRACE(10, (
"__kmpc_bound_thread_num: called\n"));
130 return __kmp_tid_from_gtid(__kmp_entry_gtid());
139 KC_TRACE(10, (
"__kmpc_bound_num_threads: called\n"));
141 return __kmp_entry_thread()->th.th_team->t.t_nproc;
161 if (__kmp_par_range == 0) {
168 semi2 = strchr(semi2,
';');
172 semi2 = strchr(semi2 + 1,
';');
176 if (__kmp_par_range_filename[0]) {
177 const char *name = semi2 - 1;
178 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
181 if ((*name ==
'/') || (*name ==
';')) {
184 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
185 return __kmp_par_range < 0;
188 semi3 = strchr(semi2 + 1,
';');
189 if (__kmp_par_range_routine[0]) {
190 if ((semi3 != NULL) && (semi3 > semi2) &&
191 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
192 return __kmp_par_range < 0;
195 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
196 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
197 return __kmp_par_range > 0;
199 return __kmp_par_range < 0;
213 return __kmp_entry_thread()->th.th_root->r.r_active;
226 kmp_int32 num_threads) {
227 KA_TRACE(20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
228 global_tid, num_threads));
230 __kmp_push_num_threads(loc, global_tid, num_threads);
233 void __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid) {
234 KA_TRACE(20, (
"__kmpc_pop_num_threads: enter\n"));
241 void __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid,
242 kmp_int32 proc_bind) {
243 KA_TRACE(20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
246 __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
262 int gtid = __kmp_entry_gtid();
264 #if (KMP_STATS_ENABLED) 268 if (previous_state == stats_state_e::SERIAL_REGION) {
269 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
271 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead);
284 va_start(ap, microtask);
287 omp_frame_t *ompt_frame;
288 if (ompt_enabled.enabled) {
289 kmp_info_t *master_th = __kmp_threads[gtid];
290 kmp_team_t *parent_team = master_th->th.th_team;
291 ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
293 ompt_frame = &(lwt->ompt_task_info.frame);
295 int tid = __kmp_tid_from_gtid(gtid);
297 parent_team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame);
299 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
300 OMPT_STORE_RETURN_ADDRESS(gtid);
304 #if INCLUDE_SSC_MARKS 307 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
308 VOLATILE_CAST(microtask_t) microtask,
309 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
311 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
317 #if INCLUDE_SSC_MARKS 320 __kmp_join_call(loc, gtid
330 #if KMP_STATS_ENABLED 331 if (previous_state == stats_state_e::SERIAL_REGION) {
332 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
334 KMP_POP_PARTITIONED_TIMER();
336 #endif // KMP_STATS_ENABLED 352 kmp_int32 num_teams, kmp_int32 num_threads) {
354 (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
355 global_tid, num_teams, num_threads));
357 __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
372 int gtid = __kmp_entry_gtid();
373 kmp_info_t *this_thr = __kmp_threads[gtid];
375 va_start(ap, microtask);
380 this_thr->th.th_teams_microtask = microtask;
381 this_thr->th.th_teams_level =
382 this_thr->th.th_team->t.t_level;
385 kmp_team_t *parent_team = this_thr->th.th_team;
386 int tid = __kmp_tid_from_gtid(gtid);
387 if (ompt_enabled.enabled) {
388 parent_team->t.t_implicit_task_taskdata[tid]
389 .ompt_task_info.frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1);
391 OMPT_STORE_RETURN_ADDRESS(gtid);
396 if (this_thr->th.th_teams_size.nteams == 0) {
397 __kmp_push_num_teams(loc, gtid, 0, 0);
399 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
400 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
401 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
403 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
404 VOLATILE_CAST(microtask_t)
406 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
407 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
413 __kmp_join_call(loc, gtid
420 this_thr->th.th_teams_microtask = NULL;
421 this_thr->th.th_teams_level = 0;
422 *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
431 int __kmpc_invoke_task_func(
int gtid) {
return __kmp_invoke_task_func(gtid); }
450 OMPT_STORE_RETURN_ADDRESS(global_tid);
452 __kmp_serialized_parallel(loc, global_tid);
463 kmp_internal_control_t *top;
464 kmp_info_t *this_thr;
465 kmp_team_t *serial_team;
468 (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
476 if (!TCR_4(__kmp_init_parallel))
477 __kmp_parallel_initialize();
479 this_thr = __kmp_threads[global_tid];
480 serial_team = this_thr->th.th_serial_team;
483 kmp_task_team_t *task_team = this_thr->th.th_task_team;
486 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)
487 __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
491 KMP_DEBUG_ASSERT(serial_team);
492 KMP_ASSERT(serial_team->t.t_serialized);
493 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
494 KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
495 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
496 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
499 if (ompt_enabled.enabled &&
500 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
501 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = NULL;
502 if (ompt_enabled.ompt_callback_implicit_task) {
503 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
504 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
505 OMPT_CUR_TASK_INFO(this_thr)->thread_num);
509 ompt_data_t *parent_task_data;
510 __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
512 if (ompt_enabled.ompt_callback_parallel_end) {
513 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
514 &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
515 ompt_invoker_program, OMPT_LOAD_RETURN_ADDRESS(global_tid));
517 __ompt_lw_taskteam_unlink(this_thr);
518 this_thr->th.ompt_thread_info.state = omp_state_overhead;
524 top = serial_team->t.t_control_stack_top;
525 if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
526 copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
527 serial_team->t.t_control_stack_top = top->next;
532 serial_team->t.t_level--;
535 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
537 dispatch_private_info_t *disp_buffer =
538 serial_team->t.t_dispatch->th_disp_buffer;
539 serial_team->t.t_dispatch->th_disp_buffer =
540 serial_team->t.t_dispatch->th_disp_buffer->next;
541 __kmp_free(disp_buffer);
544 --serial_team->t.t_serialized;
545 if (serial_team->t.t_serialized == 0) {
549 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 550 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
551 __kmp_clear_x87_fpu_status_word();
552 __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
553 __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
557 this_thr->th.th_team = serial_team->t.t_parent;
558 this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
561 this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc;
562 this_thr->th.th_team_master =
563 serial_team->t.t_parent->t.t_threads[0];
564 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
567 this_thr->th.th_dispatch =
568 &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
570 __kmp_pop_current_task_from_thread(this_thr);
572 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
573 this_thr->th.th_current_task->td_flags.executing = 1;
575 if (__kmp_tasking_mode != tskm_immediate_exec) {
577 this_thr->th.th_task_team =
578 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
580 (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / " 582 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
585 if (__kmp_tasking_mode != tskm_immediate_exec) {
586 KA_TRACE(20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting " 587 "depth of serial team %p to %d\n",
588 global_tid, serial_team, serial_team->t.t_serialized));
592 if (__kmp_env_consistency_check)
593 __kmp_pop_parallel(global_tid, NULL);
595 if (ompt_enabled.enabled)
596 this_thr->th.ompt_thread_info.state =
597 ((this_thr->th.th_team_serialized) ? omp_state_work_serial
598 : omp_state_work_parallel);
611 KC_TRACE(10, (
"__kmpc_flush: called\n"));
616 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 630 if (!__kmp_cpuinfo.initialized) {
631 __kmp_query_cpuid(&__kmp_cpuinfo);
633 if (!__kmp_cpuinfo.sse2) {
638 #elif KMP_COMPILER_MSVC 641 __sync_synchronize();
642 #endif // KMP_COMPILER_ICC 645 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64) 661 #error Unknown or unsupported architecture 664 #if OMPT_SUPPORT && OMPT_OPTIONAL 665 if (ompt_enabled.ompt_callback_flush) {
666 ompt_callbacks.ompt_callback(ompt_callback_flush)(
667 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
682 KC_TRACE(10, (
"__kmpc_barrier: called T#%d\n", global_tid));
684 if (!TCR_4(__kmp_init_parallel))
685 __kmp_parallel_initialize();
687 if (__kmp_env_consistency_check) {
689 KMP_WARNING(ConstructIdentInvalid);
692 __kmp_check_barrier(global_tid, ct_barrier, loc);
696 omp_frame_t *ompt_frame;
697 if (ompt_enabled.enabled) {
698 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
699 if (ompt_frame->enter_frame == NULL)
700 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
701 OMPT_STORE_RETURN_ADDRESS(global_tid);
704 __kmp_threads[global_tid]->th.th_ident = loc;
712 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
713 #if OMPT_SUPPORT && OMPT_OPTIONAL 714 if (ompt_enabled.enabled) {
715 ompt_frame->enter_frame = NULL;
730 KC_TRACE(10, (
"__kmpc_master: called T#%d\n", global_tid));
732 if (!TCR_4(__kmp_init_parallel))
733 __kmp_parallel_initialize();
735 if (KMP_MASTER_GTID(global_tid)) {
737 KMP_PUSH_PARTITIONED_TIMER(OMP_master);
741 #if OMPT_SUPPORT && OMPT_OPTIONAL 743 if (ompt_enabled.ompt_callback_master) {
744 kmp_info_t *this_thr = __kmp_threads[global_tid];
745 kmp_team_t *team = this_thr->th.th_team;
747 int tid = __kmp_tid_from_gtid(global_tid);
748 ompt_callbacks.ompt_callback(ompt_callback_master)(
749 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
750 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
751 OMPT_GET_RETURN_ADDRESS(0));
756 if (__kmp_env_consistency_check) {
757 #if KMP_USE_DYNAMIC_LOCK 759 __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
761 __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
764 __kmp_push_sync(global_tid, ct_master, loc, NULL);
766 __kmp_check_sync(global_tid, ct_master, loc, NULL);
782 KC_TRACE(10, (
"__kmpc_end_master: called T#%d\n", global_tid));
784 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
785 KMP_POP_PARTITIONED_TIMER();
787 #if OMPT_SUPPORT && OMPT_OPTIONAL 788 kmp_info_t *this_thr = __kmp_threads[global_tid];
789 kmp_team_t *team = this_thr->th.th_team;
790 if (ompt_enabled.ompt_callback_master) {
791 int tid = __kmp_tid_from_gtid(global_tid);
792 ompt_callbacks.ompt_callback(ompt_callback_master)(
793 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
794 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
795 OMPT_GET_RETURN_ADDRESS(0));
799 if (__kmp_env_consistency_check) {
801 KMP_WARNING(ThreadIdentInvalid);
803 if (KMP_MASTER_GTID(global_tid))
804 __kmp_pop_sync(global_tid, ct_master, loc);
818 KMP_DEBUG_ASSERT(__kmp_init_serial);
820 KC_TRACE(10, (
"__kmpc_ordered: called T#%d\n", gtid));
822 if (!TCR_4(__kmp_init_parallel))
823 __kmp_parallel_initialize();
826 __kmp_itt_ordered_prep(gtid);
830 th = __kmp_threads[gtid];
832 #if OMPT_SUPPORT && OMPT_OPTIONAL 836 if (ompt_enabled.enabled) {
837 OMPT_STORE_RETURN_ADDRESS(gtid);
838 team = __kmp_team_from_gtid(gtid);
839 lck = (omp_wait_id_t)&team->t.t_ordered.dt.t_value;
841 th->th.ompt_thread_info.wait_id = lck;
842 th->th.ompt_thread_info.state = omp_state_wait_ordered;
845 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
846 if (ompt_enabled.ompt_callback_mutex_acquire) {
847 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
848 ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin,
849 (omp_wait_id_t)lck, codeptr_ra);
854 if (th->th.th_dispatch->th_deo_fcn != 0)
855 (*th->th.th_dispatch->th_deo_fcn)(>id, &cid, loc);
857 __kmp_parallel_deo(>id, &cid, loc);
859 #if OMPT_SUPPORT && OMPT_OPTIONAL 860 if (ompt_enabled.enabled) {
862 th->th.ompt_thread_info.state = omp_state_work_parallel;
863 th->th.ompt_thread_info.wait_id = 0;
866 if (ompt_enabled.ompt_callback_mutex_acquired) {
867 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
868 ompt_mutex_ordered, (omp_wait_id_t)lck, codeptr_ra);
874 __kmp_itt_ordered_start(gtid);
889 KC_TRACE(10, (
"__kmpc_end_ordered: called T#%d\n", gtid));
892 __kmp_itt_ordered_end(gtid);
896 th = __kmp_threads[gtid];
898 if (th->th.th_dispatch->th_dxo_fcn != 0)
899 (*th->th.th_dispatch->th_dxo_fcn)(>id, &cid, loc);
901 __kmp_parallel_dxo(>id, &cid, loc);
903 #if OMPT_SUPPORT && OMPT_OPTIONAL 904 OMPT_STORE_RETURN_ADDRESS(gtid);
905 if (ompt_enabled.ompt_callback_mutex_released) {
906 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
908 (omp_wait_id_t)&__kmp_team_from_gtid(gtid)->t.t_ordered.dt.t_value,
909 OMPT_LOAD_RETURN_ADDRESS(gtid));
914 #if KMP_USE_DYNAMIC_LOCK 916 static __forceinline
void 917 __kmp_init_indirect_csptr(kmp_critical_name *crit,
ident_t const *loc,
918 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
922 kmp_indirect_lock_t **lck;
923 lck = (kmp_indirect_lock_t **)crit;
924 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
925 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
926 KMP_SET_I_LOCK_LOCATION(ilk, loc);
927 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
929 (
"__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
931 __kmp_itt_critical_creating(ilk->lock, loc);
933 int status = KMP_COMPARE_AND_STORE_PTR(lck,
nullptr, ilk);
936 __kmp_itt_critical_destroyed(ilk->lock);
942 KMP_DEBUG_ASSERT(*lck != NULL);
946 #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \ 948 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 949 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 950 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 951 if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 952 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 954 KMP_FSYNC_PREPARE(l); \ 955 KMP_INIT_YIELD(spins); \ 956 if (TCR_4(__kmp_nth) > \ 957 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 960 KMP_YIELD_SPIN(spins); \ 962 kmp_backoff_t backoff = __kmp_spin_backoff_params; \ 964 KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 965 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 966 __kmp_spin_backoff(&backoff); \ 967 if (TCR_4(__kmp_nth) > \ 968 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 971 KMP_YIELD_SPIN(spins); \ 975 KMP_FSYNC_ACQUIRED(l); \ 979 #define KMP_TEST_TAS_LOCK(lock, gtid, rc) \ 981 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 982 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 983 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 984 rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \ 985 __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \ 989 #define KMP_RELEASE_TAS_LOCK(lock, gtid) \ 990 { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); } 994 #include <sys/syscall.h> 1000 #define FUTEX_WAKE 1 1004 #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \ 1006 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1007 kmp_int32 gtid_code = (gtid + 1) << 1; \ 1009 KMP_FSYNC_PREPARE(ftx); \ 1010 kmp_int32 poll_val; \ 1011 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \ 1012 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1013 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ 1014 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ 1016 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \ 1018 KMP_LOCK_BUSY(1, futex))) { \ 1021 poll_val |= KMP_LOCK_BUSY(1, futex); \ 1024 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \ 1025 NULL, NULL, 0)) != 0) { \ 1030 KMP_FSYNC_ACQUIRED(ftx); \ 1034 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \ 1036 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1037 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1038 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \ 1039 KMP_FSYNC_ACQUIRED(ftx); \ 1047 #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \ 1049 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1051 KMP_FSYNC_RELEASING(ftx); \ 1052 kmp_int32 poll_val = \ 1053 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ 1054 if (KMP_LOCK_STRIP(poll_val) & 1) { \ 1055 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \ 1056 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ 1059 KMP_YIELD(TCR_4(__kmp_nth) > \ 1060 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ 1063 #endif // KMP_USE_FUTEX 1065 #else // KMP_USE_DYNAMIC_LOCK 1067 static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1070 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1073 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1080 lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1081 __kmp_init_user_lock_with_checks(lck);
1082 __kmp_set_user_lock_location(lck, loc);
1084 __kmp_itt_critical_creating(lck);
1095 int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1100 __kmp_itt_critical_destroyed(lck);
1104 __kmp_destroy_user_lock_with_checks(lck);
1105 __kmp_user_lock_free(&idx, gtid, lck);
1106 lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1107 KMP_DEBUG_ASSERT(lck != NULL);
1113 #endif // KMP_USE_DYNAMIC_LOCK 1126 kmp_critical_name *crit) {
1127 #if KMP_USE_DYNAMIC_LOCK 1128 #if OMPT_SUPPORT && OMPT_OPTIONAL 1129 OMPT_STORE_RETURN_ADDRESS(global_tid);
1130 #endif // OMPT_SUPPORT 1131 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1134 #if OMPT_SUPPORT && OMPT_OPTIONAL 1135 omp_state_t prev_state = omp_state_undefined;
1136 ompt_thread_info_t ti;
1138 kmp_user_lock_p lck;
1140 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1144 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1145 KMP_CHECK_USER_LOCK_INIT();
1147 if ((__kmp_user_lock_kind == lk_tas) &&
1148 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1149 lck = (kmp_user_lock_p)crit;
1152 else if ((__kmp_user_lock_kind == lk_futex) &&
1153 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1154 lck = (kmp_user_lock_p)crit;
1158 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1161 if (__kmp_env_consistency_check)
1162 __kmp_push_sync(global_tid, ct_critical, loc, lck);
1170 __kmp_itt_critical_acquiring(lck);
1172 #if OMPT_SUPPORT && OMPT_OPTIONAL 1173 OMPT_STORE_RETURN_ADDRESS(gtid);
1174 void *codeptr_ra = NULL;
1175 if (ompt_enabled.enabled) {
1176 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1178 prev_state = ti.state;
1179 ti.wait_id = (omp_wait_id_t)lck;
1180 ti.state = omp_state_wait_critical;
1183 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1184 if (ompt_enabled.ompt_callback_mutex_acquire) {
1185 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1186 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1187 (omp_wait_id_t)crit, codeptr_ra);
1193 __kmp_acquire_user_lock_with_checks(lck, global_tid);
1196 __kmp_itt_critical_acquired(lck);
1198 #if OMPT_SUPPORT && OMPT_OPTIONAL 1199 if (ompt_enabled.enabled) {
1201 ti.state = prev_state;
1205 if (ompt_enabled.ompt_callback_mutex_acquired) {
1206 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1207 ompt_mutex_critical, (omp_wait_id_t)crit, codeptr_ra);
1211 KMP_POP_PARTITIONED_TIMER();
1213 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1214 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1215 #endif // KMP_USE_DYNAMIC_LOCK 1218 #if KMP_USE_DYNAMIC_LOCK 1221 static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1223 #define KMP_TSX_LOCK(seq) lockseq_##seq 1225 #define KMP_TSX_LOCK(seq) __kmp_user_lock_seq 1228 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1229 #define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm) 1231 #define KMP_CPUINFO_RTM 0 1235 if (hint & kmp_lock_hint_hle)
1236 return KMP_TSX_LOCK(hle);
1237 if (hint & kmp_lock_hint_rtm)
1238 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm) : __kmp_user_lock_seq;
1239 if (hint & kmp_lock_hint_adaptive)
1240 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1243 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1244 return __kmp_user_lock_seq;
1245 if ((hint & omp_lock_hint_speculative) &&
1246 (hint & omp_lock_hint_nonspeculative))
1247 return __kmp_user_lock_seq;
1250 if (hint & omp_lock_hint_contended)
1251 return lockseq_queuing;
1254 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1258 if (hint & omp_lock_hint_speculative)
1259 return KMP_TSX_LOCK(hle);
1261 return __kmp_user_lock_seq;
1264 #if OMPT_SUPPORT && OMPT_OPTIONAL 1265 static kmp_mutex_impl_t
1266 __ompt_get_mutex_impl_type(
void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1268 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1273 return kmp_mutex_impl_queuing;
1276 return kmp_mutex_impl_spin;
1279 return kmp_mutex_impl_speculative;
1282 return ompt_mutex_impl_unknown;
1284 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1287 switch (ilock->type) {
1289 case locktag_adaptive:
1291 return kmp_mutex_impl_speculative;
1293 case locktag_nested_tas:
1294 return kmp_mutex_impl_spin;
1296 case locktag_nested_futex:
1298 case locktag_ticket:
1299 case locktag_queuing:
1301 case locktag_nested_ticket:
1302 case locktag_nested_queuing:
1303 case locktag_nested_drdpa:
1304 return kmp_mutex_impl_queuing;
1306 return ompt_mutex_impl_unknown;
1311 static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
1312 switch (__kmp_user_lock_kind) {
1314 return kmp_mutex_impl_spin;
1321 return kmp_mutex_impl_queuing;
1326 return kmp_mutex_impl_speculative;
1329 return ompt_mutex_impl_unknown;
1347 void __kmpc_critical_with_hint(
ident_t *loc, kmp_int32 global_tid,
1348 kmp_critical_name *crit, uintptr_t hint) {
1350 kmp_user_lock_p lck;
1351 #if OMPT_SUPPORT && OMPT_OPTIONAL 1352 omp_state_t prev_state = omp_state_undefined;
1353 ompt_thread_info_t ti;
1355 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1357 codeptr = OMPT_GET_RETURN_ADDRESS(0);
1360 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1362 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1364 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1366 kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
1367 if (KMP_IS_D_LOCK(lckseq)) {
1368 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
1369 KMP_GET_D_TAG(lckseq));
1371 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
1377 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1378 lck = (kmp_user_lock_p)lk;
1379 if (__kmp_env_consistency_check) {
1380 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1381 __kmp_map_hint_to_lock(hint));
1384 __kmp_itt_critical_acquiring(lck);
1386 #if OMPT_SUPPORT && OMPT_OPTIONAL 1387 if (ompt_enabled.enabled) {
1388 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1390 prev_state = ti.state;
1391 ti.wait_id = (omp_wait_id_t)lck;
1392 ti.state = omp_state_wait_critical;
1395 if (ompt_enabled.ompt_callback_mutex_acquire) {
1396 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1397 ompt_mutex_critical, (
unsigned int)hint,
1398 __ompt_get_mutex_impl_type(crit), (omp_wait_id_t)crit, codeptr);
1402 #if KMP_USE_INLINED_TAS 1403 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1404 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1406 #elif KMP_USE_INLINED_FUTEX 1407 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1408 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1412 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
1415 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1417 if (__kmp_env_consistency_check) {
1418 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1419 __kmp_map_hint_to_lock(hint));
1422 __kmp_itt_critical_acquiring(lck);
1424 #if OMPT_SUPPORT && OMPT_OPTIONAL 1425 if (ompt_enabled.enabled) {
1426 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1428 prev_state = ti.state;
1429 ti.wait_id = (omp_wait_id_t)lck;
1430 ti.state = omp_state_wait_critical;
1433 if (ompt_enabled.ompt_callback_mutex_acquire) {
1434 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1435 ompt_mutex_critical, (
unsigned int)hint,
1436 __ompt_get_mutex_impl_type(0, ilk), (omp_wait_id_t)crit, codeptr);
1440 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
1442 KMP_POP_PARTITIONED_TIMER();
1445 __kmp_itt_critical_acquired(lck);
1447 #if OMPT_SUPPORT && OMPT_OPTIONAL 1448 if (ompt_enabled.enabled) {
1450 ti.state = prev_state;
1454 if (ompt_enabled.ompt_callback_mutex_acquired) {
1455 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1456 ompt_mutex_critical, (omp_wait_id_t)crit, codeptr);
1461 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1462 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1465 #endif // KMP_USE_DYNAMIC_LOCK 1477 kmp_critical_name *crit) {
1478 kmp_user_lock_p lck;
1480 KC_TRACE(10, (
"__kmpc_end_critical: called T#%d\n", global_tid));
1482 #if KMP_USE_DYNAMIC_LOCK 1483 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
1484 lck = (kmp_user_lock_p)crit;
1485 KMP_ASSERT(lck != NULL);
1486 if (__kmp_env_consistency_check) {
1487 __kmp_pop_sync(global_tid, ct_critical, loc);
1490 __kmp_itt_critical_releasing(lck);
1492 #if KMP_USE_INLINED_TAS 1493 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1494 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1496 #elif KMP_USE_INLINED_FUTEX 1497 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1498 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1502 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1505 kmp_indirect_lock_t *ilk =
1506 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1507 KMP_ASSERT(ilk != NULL);
1509 if (__kmp_env_consistency_check) {
1510 __kmp_pop_sync(global_tid, ct_critical, loc);
1513 __kmp_itt_critical_releasing(lck);
1515 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1518 #else // KMP_USE_DYNAMIC_LOCK 1520 if ((__kmp_user_lock_kind == lk_tas) &&
1521 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1522 lck = (kmp_user_lock_p)crit;
1525 else if ((__kmp_user_lock_kind == lk_futex) &&
1526 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1527 lck = (kmp_user_lock_p)crit;
1531 lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit));
1534 KMP_ASSERT(lck != NULL);
1536 if (__kmp_env_consistency_check)
1537 __kmp_pop_sync(global_tid, ct_critical, loc);
1540 __kmp_itt_critical_releasing(lck);
1544 __kmp_release_user_lock_with_checks(lck, global_tid);
1546 #endif // KMP_USE_DYNAMIC_LOCK 1548 #if OMPT_SUPPORT && OMPT_OPTIONAL 1551 OMPT_STORE_RETURN_ADDRESS(global_tid);
1552 if (ompt_enabled.ompt_callback_mutex_released) {
1553 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1554 ompt_mutex_critical, (omp_wait_id_t)crit, OMPT_LOAD_RETURN_ADDRESS(0));
1558 KMP_POP_PARTITIONED_TIMER();
1559 KA_TRACE(15, (
"__kmpc_end_critical: done T#%d\n", global_tid));
1574 KC_TRACE(10, (
"__kmpc_barrier_master: called T#%d\n", global_tid));
1576 if (!TCR_4(__kmp_init_parallel))
1577 __kmp_parallel_initialize();
1579 if (__kmp_env_consistency_check)
1580 __kmp_check_barrier(global_tid, ct_barrier, loc);
1583 omp_frame_t *ompt_frame;
1584 if (ompt_enabled.enabled) {
1585 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1586 if (ompt_frame->enter_frame == NULL)
1587 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1588 OMPT_STORE_RETURN_ADDRESS(global_tid);
1592 __kmp_threads[global_tid]->th.th_ident = loc;
1594 status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL);
1595 #if OMPT_SUPPORT && OMPT_OPTIONAL 1596 if (ompt_enabled.enabled) {
1597 ompt_frame->enter_frame = NULL;
1601 return (status != 0) ? 0 : 1;
1614 KC_TRACE(10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid));
1616 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1632 KC_TRACE(10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1634 if (!TCR_4(__kmp_init_parallel))
1635 __kmp_parallel_initialize();
1637 if (__kmp_env_consistency_check) {
1639 KMP_WARNING(ConstructIdentInvalid);
1641 __kmp_check_barrier(global_tid, ct_barrier, loc);
1645 omp_frame_t *ompt_frame;
1646 if (ompt_enabled.enabled) {
1647 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1648 if (ompt_frame->enter_frame == NULL)
1649 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1650 OMPT_STORE_RETURN_ADDRESS(global_tid);
1654 __kmp_threads[global_tid]->th.th_ident = loc;
1656 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
1657 #if OMPT_SUPPORT && OMPT_OPTIONAL 1658 if (ompt_enabled.enabled) {
1659 ompt_frame->enter_frame = NULL;
1665 if (__kmp_env_consistency_check) {
1669 if (global_tid < 0) {
1670 KMP_WARNING(ThreadIdentInvalid);
1676 __kmp_pop_sync(global_tid, ct_master, loc);
1696 kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
1701 KMP_PUSH_PARTITIONED_TIMER(OMP_single);
1704 #if OMPT_SUPPORT && OMPT_OPTIONAL 1705 kmp_info_t *this_thr = __kmp_threads[global_tid];
1706 kmp_team_t *team = this_thr->th.th_team;
1707 int tid = __kmp_tid_from_gtid(global_tid);
1709 if (ompt_enabled.enabled) {
1711 if (ompt_enabled.ompt_callback_work) {
1712 ompt_callbacks.ompt_callback(ompt_callback_work)(
1713 ompt_work_single_executor, ompt_scope_begin,
1714 &(team->t.ompt_team_info.parallel_data),
1715 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1716 1, OMPT_GET_RETURN_ADDRESS(0));
1719 if (ompt_enabled.ompt_callback_work) {
1720 ompt_callbacks.ompt_callback(ompt_callback_work)(
1721 ompt_work_single_other, ompt_scope_begin,
1722 &(team->t.ompt_team_info.parallel_data),
1723 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1724 1, OMPT_GET_RETURN_ADDRESS(0));
1725 ompt_callbacks.ompt_callback(ompt_callback_work)(
1726 ompt_work_single_other, ompt_scope_end,
1727 &(team->t.ompt_team_info.parallel_data),
1728 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1729 1, OMPT_GET_RETURN_ADDRESS(0));
1748 __kmp_exit_single(global_tid);
1749 KMP_POP_PARTITIONED_TIMER();
1751 #if OMPT_SUPPORT && OMPT_OPTIONAL 1752 kmp_info_t *this_thr = __kmp_threads[global_tid];
1753 kmp_team_t *team = this_thr->th.th_team;
1754 int tid = __kmp_tid_from_gtid(global_tid);
1756 if (ompt_enabled.ompt_callback_work) {
1757 ompt_callbacks.ompt_callback(ompt_callback_work)(
1758 ompt_work_single_executor, ompt_scope_end,
1759 &(team->t.ompt_team_info.parallel_data),
1760 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
1761 OMPT_GET_RETURN_ADDRESS(0));
1774 KMP_POP_PARTITIONED_TIMER();
1775 KE_TRACE(10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1777 #if OMPT_SUPPORT && OMPT_OPTIONAL 1778 if (ompt_enabled.ompt_callback_work) {
1779 ompt_work_type_t ompt_work_type = ompt_work_loop;
1780 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
1781 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
1784 if ((loc->
flags & KMP_IDENT_WORK_LOOP) != 0) {
1785 ompt_work_type = ompt_work_loop;
1786 }
else if ((loc->
flags & KMP_IDENT_WORK_SECTIONS) != 0) {
1787 ompt_work_type = ompt_work_sections;
1788 }
else if ((loc->
flags & KMP_IDENT_WORK_DISTRIBUTE) != 0) {
1789 ompt_work_type = ompt_work_distribute;
1794 KMP_DEBUG_ASSERT(ompt_work_type);
1796 ompt_callbacks.ompt_callback(ompt_callback_work)(
1797 ompt_work_type, ompt_scope_end, &(team_info->parallel_data),
1798 &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
1801 if (__kmp_env_consistency_check)
1802 __kmp_pop_workshare(global_tid, ct_pdo, loc);
1808 void ompc_set_num_threads(
int arg) {
1810 __kmp_set_num_threads(arg, __kmp_entry_gtid());
1813 void ompc_set_dynamic(
int flag) {
1817 thread = __kmp_entry_thread();
1819 __kmp_save_internal_controls(thread);
1821 set__dynamic(thread, flag ? TRUE : FALSE);
1824 void ompc_set_nested(
int flag) {
1828 thread = __kmp_entry_thread();
1830 __kmp_save_internal_controls(thread);
1832 set__nested(thread, flag ? TRUE : FALSE);
1835 void ompc_set_max_active_levels(
int max_active_levels) {
1840 __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);
1843 void ompc_set_schedule(omp_sched_t kind,
int modifier) {
1845 __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier);
1848 int ompc_get_ancestor_thread_num(
int level) {
1849 return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level);
1852 int ompc_get_team_size(
int level) {
1853 return __kmp_get_team_size(__kmp_entry_gtid(), level);
1856 void kmpc_set_stacksize(
int arg) {
1858 __kmp_aux_set_stacksize(arg);
1861 void kmpc_set_stacksize_s(
size_t arg) {
1863 __kmp_aux_set_stacksize(arg);
1866 void kmpc_set_blocktime(
int arg) {
1870 gtid = __kmp_entry_gtid();
1871 tid = __kmp_tid_from_gtid(gtid);
1872 thread = __kmp_thread_from_gtid(gtid);
1874 __kmp_aux_set_blocktime(arg, thread, tid);
1877 void kmpc_set_library(
int arg) {
1879 __kmp_user_set_library((
enum library_type)arg);
1882 void kmpc_set_defaults(
char const *str) {
1884 __kmp_aux_set_defaults(str, KMP_STRLEN(str));
1887 void kmpc_set_disp_num_buffers(
int arg) {
1890 if (__kmp_init_serial == 0 && arg > 0)
1891 __kmp_dispatch_num_buffers = arg;
1894 int kmpc_set_affinity_mask_proc(
int proc,
void **mask) {
1895 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1898 if (!TCR_4(__kmp_init_middle)) {
1899 __kmp_middle_initialize();
1901 return __kmp_aux_set_affinity_mask_proc(proc, mask);
1905 int kmpc_unset_affinity_mask_proc(
int proc,
void **mask) {
1906 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1909 if (!TCR_4(__kmp_init_middle)) {
1910 __kmp_middle_initialize();
1912 return __kmp_aux_unset_affinity_mask_proc(proc, mask);
1916 int kmpc_get_affinity_mask_proc(
int proc,
void **mask) {
1917 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1920 if (!TCR_4(__kmp_init_middle)) {
1921 __kmp_middle_initialize();
1923 return __kmp_aux_get_affinity_mask_proc(proc, mask);
1973 void *cpy_data,
void (*cpy_func)(
void *,
void *),
1977 KC_TRACE(10, (
"__kmpc_copyprivate: called T#%d\n", gtid));
1981 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
1983 if (__kmp_env_consistency_check) {
1985 KMP_WARNING(ConstructIdentInvalid);
1992 *data_ptr = cpy_data;
1995 omp_frame_t *ompt_frame;
1996 if (ompt_enabled.enabled) {
1997 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1998 if (ompt_frame->enter_frame == NULL)
1999 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
2000 OMPT_STORE_RETURN_ADDRESS(gtid);
2005 __kmp_threads[gtid]->th.th_ident = loc;
2007 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2010 (*cpy_func)(cpy_data, *data_ptr);
2016 if (ompt_enabled.enabled) {
2017 OMPT_STORE_RETURN_ADDRESS(gtid);
2021 __kmp_threads[gtid]->th.th_ident = loc;
2024 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2025 #if OMPT_SUPPORT && OMPT_OPTIONAL 2026 if (ompt_enabled.enabled) {
2027 ompt_frame->enter_frame = NULL;
2034 #define INIT_LOCK __kmp_init_user_lock_with_checks 2035 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks 2036 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks 2037 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed 2038 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks 2039 #define ACQUIRE_NESTED_LOCK_TIMED \ 2040 __kmp_acquire_nested_user_lock_with_checks_timed 2041 #define RELEASE_LOCK __kmp_release_user_lock_with_checks 2042 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks 2043 #define TEST_LOCK __kmp_test_user_lock_with_checks 2044 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks 2045 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks 2046 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks 2051 #if KMP_USE_DYNAMIC_LOCK 2054 static __forceinline
void __kmp_init_lock_with_hint(
ident_t *loc,
void **lock,
2055 kmp_dyna_lockseq_t seq) {
2056 if (KMP_IS_D_LOCK(seq)) {
2057 KMP_INIT_D_LOCK(lock, seq);
2059 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
2062 KMP_INIT_I_LOCK(lock, seq);
2064 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2065 __kmp_itt_lock_creating(ilk->lock, loc);
2071 static __forceinline
void 2072 __kmp_init_nest_lock_with_hint(
ident_t *loc,
void **lock,
2073 kmp_dyna_lockseq_t seq) {
2076 if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
2077 seq = __kmp_user_lock_seq;
2081 seq = lockseq_nested_tas;
2085 seq = lockseq_nested_futex;
2088 case lockseq_ticket:
2089 seq = lockseq_nested_ticket;
2091 case lockseq_queuing:
2092 seq = lockseq_nested_queuing;
2095 seq = lockseq_nested_drdpa;
2098 seq = lockseq_nested_queuing;
2100 KMP_INIT_I_LOCK(lock, seq);
2102 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2103 __kmp_itt_lock_creating(ilk->lock, loc);
2108 void __kmpc_init_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
void **user_lock,
2110 KMP_DEBUG_ASSERT(__kmp_init_serial);
2111 if (__kmp_env_consistency_check && user_lock == NULL) {
2112 KMP_FATAL(LockIsUninitialized,
"omp_init_lock_with_hint");
2115 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2117 #if OMPT_SUPPORT && OMPT_OPTIONAL 2119 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2121 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2122 if (ompt_enabled.ompt_callback_lock_init) {
2123 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2124 ompt_mutex_lock, (omp_lock_hint_t)hint,
2125 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2132 void __kmpc_init_nest_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
2133 void **user_lock, uintptr_t hint) {
2134 KMP_DEBUG_ASSERT(__kmp_init_serial);
2135 if (__kmp_env_consistency_check && user_lock == NULL) {
2136 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock_with_hint");
2139 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2141 #if OMPT_SUPPORT && OMPT_OPTIONAL 2143 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2145 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2146 if (ompt_enabled.ompt_callback_lock_init) {
2147 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2148 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2149 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2155 #endif // KMP_USE_DYNAMIC_LOCK 2158 void __kmpc_init_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2159 #if KMP_USE_DYNAMIC_LOCK 2161 KMP_DEBUG_ASSERT(__kmp_init_serial);
2162 if (__kmp_env_consistency_check && user_lock == NULL) {
2163 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
2165 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2167 #if OMPT_SUPPORT && OMPT_OPTIONAL 2169 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2171 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2172 if (ompt_enabled.ompt_callback_lock_init) {
2173 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2174 ompt_mutex_lock, omp_lock_hint_none,
2175 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2180 #else // KMP_USE_DYNAMIC_LOCK 2182 static char const *
const func =
"omp_init_lock";
2183 kmp_user_lock_p lck;
2184 KMP_DEBUG_ASSERT(__kmp_init_serial);
2186 if (__kmp_env_consistency_check) {
2187 if (user_lock == NULL) {
2188 KMP_FATAL(LockIsUninitialized, func);
2192 KMP_CHECK_USER_LOCK_INIT();
2194 if ((__kmp_user_lock_kind == lk_tas) &&
2195 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2196 lck = (kmp_user_lock_p)user_lock;
2199 else if ((__kmp_user_lock_kind == lk_futex) &&
2200 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2201 lck = (kmp_user_lock_p)user_lock;
2205 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2208 __kmp_set_user_lock_location(lck, loc);
2210 #if OMPT_SUPPORT && OMPT_OPTIONAL 2212 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2214 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2215 if (ompt_enabled.ompt_callback_lock_init) {
2216 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2217 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2218 (omp_wait_id_t)user_lock, codeptr);
2223 __kmp_itt_lock_creating(lck);
2226 #endif // KMP_USE_DYNAMIC_LOCK 2230 void __kmpc_init_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2231 #if KMP_USE_DYNAMIC_LOCK 2233 KMP_DEBUG_ASSERT(__kmp_init_serial);
2234 if (__kmp_env_consistency_check && user_lock == NULL) {
2235 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
2237 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2239 #if OMPT_SUPPORT && OMPT_OPTIONAL 2241 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2243 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2244 if (ompt_enabled.ompt_callback_lock_init) {
2245 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2246 ompt_mutex_nest_lock, omp_lock_hint_none,
2247 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2252 #else // KMP_USE_DYNAMIC_LOCK 2254 static char const *
const func =
"omp_init_nest_lock";
2255 kmp_user_lock_p lck;
2256 KMP_DEBUG_ASSERT(__kmp_init_serial);
2258 if (__kmp_env_consistency_check) {
2259 if (user_lock == NULL) {
2260 KMP_FATAL(LockIsUninitialized, func);
2264 KMP_CHECK_USER_LOCK_INIT();
2266 if ((__kmp_user_lock_kind == lk_tas) &&
2267 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2268 OMP_NEST_LOCK_T_SIZE)) {
2269 lck = (kmp_user_lock_p)user_lock;
2272 else if ((__kmp_user_lock_kind == lk_futex) &&
2273 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2274 OMP_NEST_LOCK_T_SIZE)) {
2275 lck = (kmp_user_lock_p)user_lock;
2279 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2282 INIT_NESTED_LOCK(lck);
2283 __kmp_set_user_lock_location(lck, loc);
2285 #if OMPT_SUPPORT && OMPT_OPTIONAL 2287 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2289 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2290 if (ompt_enabled.ompt_callback_lock_init) {
2291 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2292 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2293 (omp_wait_id_t)user_lock, codeptr);
2298 __kmp_itt_lock_creating(lck);
2301 #endif // KMP_USE_DYNAMIC_LOCK 2304 void __kmpc_destroy_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2305 #if KMP_USE_DYNAMIC_LOCK 2308 kmp_user_lock_p lck;
2309 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2310 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2312 lck = (kmp_user_lock_p)user_lock;
2314 __kmp_itt_lock_destroyed(lck);
2316 #if OMPT_SUPPORT && OMPT_OPTIONAL 2318 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2320 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2321 if (ompt_enabled.ompt_callback_lock_destroy) {
2322 kmp_user_lock_p lck;
2323 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2324 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2326 lck = (kmp_user_lock_p)user_lock;
2328 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2329 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2332 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2334 kmp_user_lock_p lck;
2336 if ((__kmp_user_lock_kind == lk_tas) &&
2337 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2338 lck = (kmp_user_lock_p)user_lock;
2341 else if ((__kmp_user_lock_kind == lk_futex) &&
2342 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2343 lck = (kmp_user_lock_p)user_lock;
2347 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_lock");
2350 #if OMPT_SUPPORT && OMPT_OPTIONAL 2352 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2354 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2355 if (ompt_enabled.ompt_callback_lock_destroy) {
2356 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2357 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2362 __kmp_itt_lock_destroyed(lck);
2366 if ((__kmp_user_lock_kind == lk_tas) &&
2367 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2371 else if ((__kmp_user_lock_kind == lk_futex) &&
2372 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2377 __kmp_user_lock_free(user_lock, gtid, lck);
2379 #endif // KMP_USE_DYNAMIC_LOCK 2383 void __kmpc_destroy_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2384 #if KMP_USE_DYNAMIC_LOCK 2387 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2388 __kmp_itt_lock_destroyed(ilk->lock);
2390 #if OMPT_SUPPORT && OMPT_OPTIONAL 2392 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2394 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2395 if (ompt_enabled.ompt_callback_lock_destroy) {
2396 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2397 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2400 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2402 #else // KMP_USE_DYNAMIC_LOCK 2404 kmp_user_lock_p lck;
2406 if ((__kmp_user_lock_kind == lk_tas) &&
2407 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2408 OMP_NEST_LOCK_T_SIZE)) {
2409 lck = (kmp_user_lock_p)user_lock;
2412 else if ((__kmp_user_lock_kind == lk_futex) &&
2413 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2414 OMP_NEST_LOCK_T_SIZE)) {
2415 lck = (kmp_user_lock_p)user_lock;
2419 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_nest_lock");
2422 #if OMPT_SUPPORT && OMPT_OPTIONAL 2424 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2426 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2427 if (ompt_enabled.ompt_callback_lock_destroy) {
2428 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2429 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2434 __kmp_itt_lock_destroyed(lck);
2437 DESTROY_NESTED_LOCK(lck);
2439 if ((__kmp_user_lock_kind == lk_tas) &&
2440 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2441 OMP_NEST_LOCK_T_SIZE)) {
2445 else if ((__kmp_user_lock_kind == lk_futex) &&
2446 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2447 OMP_NEST_LOCK_T_SIZE)) {
2452 __kmp_user_lock_free(user_lock, gtid, lck);
2454 #endif // KMP_USE_DYNAMIC_LOCK 2457 void __kmpc_set_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2459 #if KMP_USE_DYNAMIC_LOCK 2460 int tag = KMP_EXTRACT_D_TAG(user_lock);
2462 __kmp_itt_lock_acquiring(
2466 #if OMPT_SUPPORT && OMPT_OPTIONAL 2468 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2470 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2471 if (ompt_enabled.ompt_callback_mutex_acquire) {
2472 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2473 ompt_mutex_lock, omp_lock_hint_none,
2474 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2478 #if KMP_USE_INLINED_TAS 2479 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2480 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2482 #elif KMP_USE_INLINED_FUTEX 2483 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2484 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2488 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2491 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2493 #if OMPT_SUPPORT && OMPT_OPTIONAL 2494 if (ompt_enabled.ompt_callback_mutex_acquired) {
2495 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2496 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2500 #else // KMP_USE_DYNAMIC_LOCK 2502 kmp_user_lock_p lck;
2504 if ((__kmp_user_lock_kind == lk_tas) &&
2505 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2506 lck = (kmp_user_lock_p)user_lock;
2509 else if ((__kmp_user_lock_kind == lk_futex) &&
2510 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2511 lck = (kmp_user_lock_p)user_lock;
2515 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_lock");
2519 __kmp_itt_lock_acquiring(lck);
2521 #if OMPT_SUPPORT && OMPT_OPTIONAL 2523 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2525 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2526 if (ompt_enabled.ompt_callback_mutex_acquire) {
2527 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2528 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2529 (omp_wait_id_t)lck, codeptr);
2533 ACQUIRE_LOCK(lck, gtid);
2536 __kmp_itt_lock_acquired(lck);
2539 #if OMPT_SUPPORT && OMPT_OPTIONAL 2540 if (ompt_enabled.ompt_callback_mutex_acquired) {
2541 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2542 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2546 #endif // KMP_USE_DYNAMIC_LOCK 2549 void __kmpc_set_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2550 #if KMP_USE_DYNAMIC_LOCK 2553 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2555 #if OMPT_SUPPORT && OMPT_OPTIONAL 2557 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2559 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2560 if (ompt_enabled.enabled) {
2561 if (ompt_enabled.ompt_callback_mutex_acquire) {
2562 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2563 ompt_mutex_nest_lock, omp_lock_hint_none,
2564 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2569 int acquire_status =
2570 KMP_D_LOCK_FUNC(user_lock,
set)((kmp_dyna_lock_t *)user_lock, gtid);
2572 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2575 #if OMPT_SUPPORT && OMPT_OPTIONAL 2576 if (ompt_enabled.enabled) {
2577 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2578 if (ompt_enabled.ompt_callback_mutex_acquired) {
2580 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2581 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2584 if (ompt_enabled.ompt_callback_nest_lock) {
2586 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2587 ompt_scope_begin, (omp_wait_id_t)user_lock, codeptr);
2593 #else // KMP_USE_DYNAMIC_LOCK 2595 kmp_user_lock_p lck;
2597 if ((__kmp_user_lock_kind == lk_tas) &&
2598 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2599 OMP_NEST_LOCK_T_SIZE)) {
2600 lck = (kmp_user_lock_p)user_lock;
2603 else if ((__kmp_user_lock_kind == lk_futex) &&
2604 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2605 OMP_NEST_LOCK_T_SIZE)) {
2606 lck = (kmp_user_lock_p)user_lock;
2610 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_nest_lock");
2614 __kmp_itt_lock_acquiring(lck);
2616 #if OMPT_SUPPORT && OMPT_OPTIONAL 2618 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2620 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2621 if (ompt_enabled.enabled) {
2622 if (ompt_enabled.ompt_callback_mutex_acquire) {
2623 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2624 ompt_mutex_nest_lock, omp_lock_hint_none,
2625 __ompt_get_mutex_impl_type(), (omp_wait_id_t)lck, codeptr);
2630 ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status);
2633 __kmp_itt_lock_acquired(lck);
2636 #if OMPT_SUPPORT && OMPT_OPTIONAL 2637 if (ompt_enabled.enabled) {
2638 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2639 if (ompt_enabled.ompt_callback_mutex_acquired) {
2641 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2642 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
2645 if (ompt_enabled.ompt_callback_nest_lock) {
2647 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2648 ompt_scope_begin, (omp_wait_id_t)lck, codeptr);
2654 #endif // KMP_USE_DYNAMIC_LOCK 2657 void __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2658 #if KMP_USE_DYNAMIC_LOCK 2660 int tag = KMP_EXTRACT_D_TAG(user_lock);
2662 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2664 #if KMP_USE_INLINED_TAS 2665 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2666 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
2668 #elif KMP_USE_INLINED_FUTEX 2669 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2670 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
2674 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2677 #if OMPT_SUPPORT && OMPT_OPTIONAL 2679 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2681 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2682 if (ompt_enabled.ompt_callback_mutex_released) {
2683 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2684 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2688 #else // KMP_USE_DYNAMIC_LOCK 2690 kmp_user_lock_p lck;
2695 if ((__kmp_user_lock_kind == lk_tas) &&
2696 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2697 #if KMP_OS_LINUX && \ 2698 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2701 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2703 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2706 #if OMPT_SUPPORT && OMPT_OPTIONAL 2708 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2710 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2711 if (ompt_enabled.ompt_callback_mutex_released) {
2712 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2713 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2719 lck = (kmp_user_lock_p)user_lock;
2723 else if ((__kmp_user_lock_kind == lk_futex) &&
2724 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2725 lck = (kmp_user_lock_p)user_lock;
2729 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_lock");
2733 __kmp_itt_lock_releasing(lck);
2736 RELEASE_LOCK(lck, gtid);
2738 #if OMPT_SUPPORT && OMPT_OPTIONAL 2740 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2742 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2743 if (ompt_enabled.ompt_callback_mutex_released) {
2744 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2745 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2749 #endif // KMP_USE_DYNAMIC_LOCK 2753 void __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2754 #if KMP_USE_DYNAMIC_LOCK 2757 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2759 int release_status =
2760 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2762 #if OMPT_SUPPORT && OMPT_OPTIONAL 2764 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2766 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2767 if (ompt_enabled.enabled) {
2768 if (release_status == KMP_LOCK_RELEASED) {
2769 if (ompt_enabled.ompt_callback_mutex_released) {
2771 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2772 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
2774 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2776 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2777 ompt_scope_end, (omp_wait_id_t)user_lock, codeptr);
2782 #else // KMP_USE_DYNAMIC_LOCK 2784 kmp_user_lock_p lck;
2788 if ((__kmp_user_lock_kind == lk_tas) &&
2789 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2790 OMP_NEST_LOCK_T_SIZE)) {
2791 #if KMP_OS_LINUX && \ 2792 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2794 kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock;
2796 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2799 #if OMPT_SUPPORT && OMPT_OPTIONAL 2800 int release_status = KMP_LOCK_STILL_HELD;
2803 if (--(tl->lk.depth_locked) == 0) {
2804 TCW_4(tl->lk.poll, 0);
2805 #if OMPT_SUPPORT && OMPT_OPTIONAL 2806 release_status = KMP_LOCK_RELEASED;
2811 #if OMPT_SUPPORT && OMPT_OPTIONAL 2813 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2815 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2816 if (ompt_enabled.enabled) {
2817 if (release_status == KMP_LOCK_RELEASED) {
2818 if (ompt_enabled.ompt_callback_mutex_released) {
2820 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2821 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
2823 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2825 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2826 ompt_mutex_scope_end, (omp_wait_id_t)lck, codeptr);
2833 lck = (kmp_user_lock_p)user_lock;
2837 else if ((__kmp_user_lock_kind == lk_futex) &&
2838 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2839 OMP_NEST_LOCK_T_SIZE)) {
2840 lck = (kmp_user_lock_p)user_lock;
2844 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_nest_lock");
2848 __kmp_itt_lock_releasing(lck);
2852 release_status = RELEASE_NESTED_LOCK(lck, gtid);
2853 #if OMPT_SUPPORT && OMPT_OPTIONAL 2855 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2857 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2858 if (ompt_enabled.enabled) {
2859 if (release_status == KMP_LOCK_RELEASED) {
2860 if (ompt_enabled.ompt_callback_mutex_released) {
2862 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2863 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
2865 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2867 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2868 ompt_mutex_scope_end, (omp_wait_id_t)lck, codeptr);
2873 #endif // KMP_USE_DYNAMIC_LOCK 2877 int __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2880 #if KMP_USE_DYNAMIC_LOCK 2882 int tag = KMP_EXTRACT_D_TAG(user_lock);
2884 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2886 #if OMPT_SUPPORT && OMPT_OPTIONAL 2888 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2890 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2891 if (ompt_enabled.ompt_callback_mutex_acquire) {
2892 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2893 ompt_mutex_lock, omp_lock_hint_none,
2894 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
2898 #if KMP_USE_INLINED_TAS 2899 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2900 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
2902 #elif KMP_USE_INLINED_FUTEX 2903 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2904 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
2908 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2912 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2914 #if OMPT_SUPPORT && OMPT_OPTIONAL 2915 if (ompt_enabled.ompt_callback_mutex_acquired) {
2916 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2917 ompt_mutex_lock, (omp_wait_id_t)user_lock, codeptr);
2923 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2928 #else // KMP_USE_DYNAMIC_LOCK 2930 kmp_user_lock_p lck;
2933 if ((__kmp_user_lock_kind == lk_tas) &&
2934 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2935 lck = (kmp_user_lock_p)user_lock;
2938 else if ((__kmp_user_lock_kind == lk_futex) &&
2939 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2940 lck = (kmp_user_lock_p)user_lock;
2944 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_lock");
2948 __kmp_itt_lock_acquiring(lck);
2950 #if OMPT_SUPPORT && OMPT_OPTIONAL 2952 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2954 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2955 if (ompt_enabled.ompt_callback_mutex_acquire) {
2956 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2957 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2958 (omp_wait_id_t)lck, codeptr);
2962 rc = TEST_LOCK(lck, gtid);
2965 __kmp_itt_lock_acquired(lck);
2967 __kmp_itt_lock_cancelled(lck);
2970 #if OMPT_SUPPORT && OMPT_OPTIONAL 2971 if (rc && ompt_enabled.ompt_callback_mutex_acquired) {
2972 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2973 ompt_mutex_lock, (omp_wait_id_t)lck, codeptr);
2977 return (rc ? FTN_TRUE : FTN_FALSE);
2981 #endif // KMP_USE_DYNAMIC_LOCK 2985 int __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2986 #if KMP_USE_DYNAMIC_LOCK 2989 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2991 #if OMPT_SUPPORT && OMPT_OPTIONAL 2993 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2995 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2996 if (ompt_enabled.ompt_callback_mutex_acquire) {
2997 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2998 ompt_mutex_nest_lock, omp_lock_hint_none,
2999 __ompt_get_mutex_impl_type(user_lock), (omp_wait_id_t)user_lock,
3003 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
3006 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3008 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3011 #if OMPT_SUPPORT && OMPT_OPTIONAL 3012 if (ompt_enabled.enabled && rc) {
3014 if (ompt_enabled.ompt_callback_mutex_acquired) {
3016 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3017 ompt_mutex_nest_lock, (omp_wait_id_t)user_lock, codeptr);
3020 if (ompt_enabled.ompt_callback_nest_lock) {
3022 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3023 ompt_scope_begin, (omp_wait_id_t)user_lock, codeptr);
3030 #else // KMP_USE_DYNAMIC_LOCK 3032 kmp_user_lock_p lck;
3035 if ((__kmp_user_lock_kind == lk_tas) &&
3036 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
3037 OMP_NEST_LOCK_T_SIZE)) {
3038 lck = (kmp_user_lock_p)user_lock;
3041 else if ((__kmp_user_lock_kind == lk_futex) &&
3042 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
3043 OMP_NEST_LOCK_T_SIZE)) {
3044 lck = (kmp_user_lock_p)user_lock;
3048 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_nest_lock");
3052 __kmp_itt_lock_acquiring(lck);
3055 #if OMPT_SUPPORT && OMPT_OPTIONAL 3057 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3059 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3060 if (ompt_enabled.enabled) &&
3061 ompt_enabled.ompt_callback_mutex_acquire) {
3062 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3063 ompt_mutex_nest_lock, omp_lock_hint_none,
3064 __ompt_get_mutex_impl_type(), (omp_wait_id_t)lck, codeptr);
3068 rc = TEST_NESTED_LOCK(lck, gtid);
3071 __kmp_itt_lock_acquired(lck);
3073 __kmp_itt_lock_cancelled(lck);
3076 #if OMPT_SUPPORT && OMPT_OPTIONAL 3077 if (ompt_enabled.enabled && rc) {
3079 if (ompt_enabled.ompt_callback_mutex_acquired) {
3081 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3082 ompt_mutex_nest_lock, (omp_wait_id_t)lck, codeptr);
3085 if (ompt_enabled.ompt_callback_nest_lock) {
3087 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3088 ompt_mutex_scope_begin, (omp_wait_id_t)lck, codeptr);
3097 #endif // KMP_USE_DYNAMIC_LOCK 3107 #define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \ 3108 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod)) 3110 #define __KMP_GET_REDUCTION_METHOD(gtid) \ 3111 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) 3117 static __forceinline
void 3118 __kmp_enter_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3119 kmp_critical_name *crit) {
3125 kmp_user_lock_p lck;
3127 #if KMP_USE_DYNAMIC_LOCK 3129 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
3132 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3133 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
3134 KMP_GET_D_TAG(__kmp_user_lock_seq));
3136 __kmp_init_indirect_csptr(crit, loc, global_tid,
3137 KMP_GET_I_TAG(__kmp_user_lock_seq));
3143 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3144 lck = (kmp_user_lock_p)lk;
3145 KMP_DEBUG_ASSERT(lck != NULL);
3146 if (__kmp_env_consistency_check) {
3147 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3149 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
3151 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3153 KMP_DEBUG_ASSERT(lck != NULL);
3154 if (__kmp_env_consistency_check) {
3155 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3157 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
3160 #else // KMP_USE_DYNAMIC_LOCK 3165 if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) {
3166 lck = (kmp_user_lock_p)crit;
3168 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
3170 KMP_DEBUG_ASSERT(lck != NULL);
3172 if (__kmp_env_consistency_check)
3173 __kmp_push_sync(global_tid, ct_critical, loc, lck);
3175 __kmp_acquire_user_lock_with_checks(lck, global_tid);
3177 #endif // KMP_USE_DYNAMIC_LOCK 3181 static __forceinline
void 3182 __kmp_end_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3183 kmp_critical_name *crit) {
3185 kmp_user_lock_p lck;
3187 #if KMP_USE_DYNAMIC_LOCK 3189 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3190 lck = (kmp_user_lock_p)crit;
3191 if (__kmp_env_consistency_check)
3192 __kmp_pop_sync(global_tid, ct_critical, loc);
3193 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
3195 kmp_indirect_lock_t *ilk =
3196 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
3197 if (__kmp_env_consistency_check)
3198 __kmp_pop_sync(global_tid, ct_critical, loc);
3199 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3202 #else // KMP_USE_DYNAMIC_LOCK 3207 if (__kmp_base_user_lock_size > 32) {
3208 lck = *((kmp_user_lock_p *)crit);
3209 KMP_ASSERT(lck != NULL);
3211 lck = (kmp_user_lock_p)crit;
3214 if (__kmp_env_consistency_check)
3215 __kmp_pop_sync(global_tid, ct_critical, loc);
3217 __kmp_release_user_lock_with_checks(lck, global_tid);
3219 #endif // KMP_USE_DYNAMIC_LOCK 3223 static __forceinline
int 3224 __kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p,
3229 if (th->th.th_teams_microtask) {
3230 *team_p = team = th->th.th_team;
3231 if (team->t.t_level == th->th.th_teams_level) {
3233 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
3235 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
3236 th->th.th_team = team->t.t_parent;
3237 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3238 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3239 *task_state = th->th.th_task_state;
3240 th->th.th_task_state = 0;
3248 static __forceinline
void 3249 __kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team,
int task_state) {
3251 th->th.th_info.ds.ds_tid = 0;
3252 th->th.th_team = team;
3253 th->th.th_team_nproc = team->t.t_nproc;
3254 th->th.th_task_team = team->t.t_task_team[task_state];
3255 th->th.th_task_state = task_state;
3277 size_t reduce_size,
void *reduce_data,
3278 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3279 kmp_critical_name *lck) {
3283 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3287 int teams_swapped = 0, task_state;
3289 KA_TRACE(10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3297 if (!TCR_4(__kmp_init_parallel))
3298 __kmp_parallel_initialize();
3301 #if KMP_USE_DYNAMIC_LOCK 3302 if (__kmp_env_consistency_check)
3303 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3305 if (__kmp_env_consistency_check)
3306 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3310 th = __kmp_thread_from_gtid(global_tid);
3311 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3312 #endif // OMP_40_ENABLED 3330 packed_reduction_method = __kmp_determine_reduction_method(
3331 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3332 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3334 if (packed_reduction_method == critical_reduce_block) {
3336 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3339 }
else if (packed_reduction_method == empty_reduce_block) {
3345 }
else if (packed_reduction_method == atomic_reduce_block) {
3355 if (__kmp_env_consistency_check)
3356 __kmp_pop_sync(global_tid, ct_reduce, loc);
3358 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3359 tree_reduce_block)) {
3379 omp_frame_t *ompt_frame;
3380 if (ompt_enabled.enabled) {
3381 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3382 if (ompt_frame->enter_frame == NULL)
3383 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3384 OMPT_STORE_RETURN_ADDRESS(global_tid);
3388 __kmp_threads[global_tid]->th.th_ident = loc;
3391 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3392 global_tid, FALSE, reduce_size, reduce_data, reduce_func);
3393 retval = (retval != 0) ? (0) : (1);
3394 #if OMPT_SUPPORT && OMPT_OPTIONAL 3395 if (ompt_enabled.enabled) {
3396 ompt_frame->enter_frame = NULL;
3402 if (__kmp_env_consistency_check) {
3404 __kmp_pop_sync(global_tid, ct_reduce, loc);
3414 if (teams_swapped) {
3415 __kmp_restore_swapped_teams(th, team, task_state);
3420 (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3421 global_tid, packed_reduction_method, retval));
3435 kmp_critical_name *lck) {
3437 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3439 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3441 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3443 if (packed_reduction_method == critical_reduce_block) {
3445 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3447 }
else if (packed_reduction_method == empty_reduce_block) {
3452 }
else if (packed_reduction_method == atomic_reduce_block) {
3459 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3460 tree_reduce_block)) {
3470 if (__kmp_env_consistency_check)
3471 __kmp_pop_sync(global_tid, ct_reduce, loc);
3473 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3474 global_tid, packed_reduction_method));
3497 size_t reduce_size,
void *reduce_data,
3498 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3499 kmp_critical_name *lck) {
3502 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3506 int teams_swapped = 0, task_state;
3509 KA_TRACE(10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid));
3517 if (!TCR_4(__kmp_init_parallel))
3518 __kmp_parallel_initialize();
3521 #if KMP_USE_DYNAMIC_LOCK 3522 if (__kmp_env_consistency_check)
3523 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3525 if (__kmp_env_consistency_check)
3526 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3530 th = __kmp_thread_from_gtid(global_tid);
3531 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3532 #endif // OMP_40_ENABLED 3534 packed_reduction_method = __kmp_determine_reduction_method(
3535 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3536 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3538 if (packed_reduction_method == critical_reduce_block) {
3540 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3543 }
else if (packed_reduction_method == empty_reduce_block) {
3549 }
else if (packed_reduction_method == atomic_reduce_block) {
3553 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3554 tree_reduce_block)) {
3560 omp_frame_t *ompt_frame;
3561 if (ompt_enabled.enabled) {
3562 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3563 if (ompt_frame->enter_frame == NULL)
3564 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3565 OMPT_STORE_RETURN_ADDRESS(global_tid);
3569 __kmp_threads[global_tid]->th.th_ident =
3573 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3574 global_tid, TRUE, reduce_size, reduce_data, reduce_func);
3575 retval = (retval != 0) ? (0) : (1);
3576 #if OMPT_SUPPORT && OMPT_OPTIONAL 3577 if (ompt_enabled.enabled) {
3578 ompt_frame->enter_frame = NULL;
3584 if (__kmp_env_consistency_check) {
3586 __kmp_pop_sync(global_tid, ct_reduce, loc);
3596 if (teams_swapped) {
3597 __kmp_restore_swapped_teams(th, team, task_state);
3602 (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
3603 global_tid, packed_reduction_method, retval));
3619 kmp_critical_name *lck) {
3621 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3625 int teams_swapped = 0, task_state;
3628 KA_TRACE(10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid));
3631 th = __kmp_thread_from_gtid(global_tid);
3632 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3633 #endif // OMP_40_ENABLED 3635 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3640 if (packed_reduction_method == critical_reduce_block) {
3642 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3646 omp_frame_t *ompt_frame;
3647 if (ompt_enabled.enabled) {
3648 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3649 if (ompt_frame->enter_frame == NULL)
3650 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3651 OMPT_STORE_RETURN_ADDRESS(global_tid);
3655 __kmp_threads[global_tid]->th.th_ident = loc;
3657 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3658 #if OMPT_SUPPORT && OMPT_OPTIONAL 3659 if (ompt_enabled.enabled) {
3660 ompt_frame->enter_frame = NULL;
3664 }
else if (packed_reduction_method == empty_reduce_block) {
3670 omp_frame_t *ompt_frame;
3671 if (ompt_enabled.enabled) {
3672 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3673 if (ompt_frame->enter_frame == NULL)
3674 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3675 OMPT_STORE_RETURN_ADDRESS(global_tid);
3679 __kmp_threads[global_tid]->th.th_ident = loc;
3681 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3682 #if OMPT_SUPPORT && OMPT_OPTIONAL 3683 if (ompt_enabled.enabled) {
3684 ompt_frame->enter_frame = NULL;
3688 }
else if (packed_reduction_method == atomic_reduce_block) {
3691 omp_frame_t *ompt_frame;
3692 if (ompt_enabled.enabled) {
3693 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3694 if (ompt_frame->enter_frame == NULL)
3695 ompt_frame->enter_frame = OMPT_GET_FRAME_ADDRESS(1);
3696 OMPT_STORE_RETURN_ADDRESS(global_tid);
3701 __kmp_threads[global_tid]->th.th_ident = loc;
3703 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3704 #if OMPT_SUPPORT && OMPT_OPTIONAL 3705 if (ompt_enabled.enabled) {
3706 ompt_frame->enter_frame = NULL;
3710 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3711 tree_reduce_block)) {
3714 __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3723 if (teams_swapped) {
3724 __kmp_restore_swapped_teams(th, team, task_state);
3728 if (__kmp_env_consistency_check)
3729 __kmp_pop_sync(global_tid, ct_reduce, loc);
3731 KA_TRACE(10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n",
3732 global_tid, packed_reduction_method));
3737 #undef __KMP_GET_REDUCTION_METHOD 3738 #undef __KMP_SET_REDUCTION_METHOD 3742 kmp_uint64 __kmpc_get_taskid() {
3747 gtid = __kmp_get_gtid();
3751 thread = __kmp_thread_from_gtid(gtid);
3752 return thread->th.th_current_task->td_task_id;
3756 kmp_uint64 __kmpc_get_parent_taskid() {
3760 kmp_taskdata_t *parent_task;
3762 gtid = __kmp_get_gtid();
3766 thread = __kmp_thread_from_gtid(gtid);
3767 parent_task = thread->th.th_current_task->td_parent;
3768 return (parent_task == NULL ? 0 : parent_task->td_task_id);
3784 void __kmpc_doacross_init(
ident_t *loc,
int gtid,
int num_dims,
3785 const struct kmp_dim *dims) {
3787 kmp_int64 last, trace_count;
3788 kmp_info_t *th = __kmp_threads[gtid];
3789 kmp_team_t *team = th->th.th_team;
3791 kmp_disp_t *pr_buf = th->th.th_dispatch;
3792 dispatch_shared_info_t *sh_buf;
3796 (
"__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
3797 gtid, num_dims, !team->t.t_serialized));
3798 KMP_DEBUG_ASSERT(dims != NULL);
3799 KMP_DEBUG_ASSERT(num_dims > 0);
3801 if (team->t.t_serialized) {
3802 KA_TRACE(20, (
"__kmpc_doacross_init() exit: serialized team\n"));
3805 KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
3806 idx = pr_buf->th_doacross_buf_idx++;
3808 sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
3811 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
3812 pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc(
3813 th,
sizeof(kmp_int64) * (4 * num_dims + 1));
3814 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3815 pr_buf->th_doacross_info[0] =
3816 (kmp_int64)num_dims;
3819 pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
3820 pr_buf->th_doacross_info[2] = dims[0].lo;
3821 pr_buf->th_doacross_info[3] = dims[0].up;
3822 pr_buf->th_doacross_info[4] = dims[0].st;
3824 for (j = 1; j < num_dims; ++j) {
3827 if (dims[j].st == 1) {
3829 range_length = dims[j].up - dims[j].lo + 1;
3831 if (dims[j].st > 0) {
3832 KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
3833 range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
3835 KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
3837 (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
3840 pr_buf->th_doacross_info[last++] = range_length;
3841 pr_buf->th_doacross_info[last++] = dims[j].lo;
3842 pr_buf->th_doacross_info[last++] = dims[j].up;
3843 pr_buf->th_doacross_info[last++] = dims[j].st;
3848 if (dims[0].st == 1) {
3849 trace_count = dims[0].up - dims[0].lo + 1;
3850 }
else if (dims[0].st > 0) {
3851 KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
3852 trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
3854 KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
3855 trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
3857 for (j = 1; j < num_dims; ++j) {
3858 trace_count *= pr_buf->th_doacross_info[4 * j + 1];
3860 KMP_DEBUG_ASSERT(trace_count > 0);
3864 if (idx != sh_buf->doacross_buf_idx) {
3866 __kmp_wait_yield_4((
volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
3873 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
3874 (
volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
3876 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
3877 (
volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
3879 if (flags == NULL) {
3881 size_t size = trace_count / 8 + 8;
3882 flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
3884 sh_buf->doacross_flags = flags;
3885 }
else if (flags == (kmp_uint32 *)1) {
3888 while (*(
volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
3890 while (*(
volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
3897 KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1);
3898 pr_buf->th_doacross_flags =
3899 sh_buf->doacross_flags;
3901 KA_TRACE(20, (
"__kmpc_doacross_init() exit: T#%d\n", gtid));
3904 void __kmpc_doacross_wait(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
3905 kmp_int32 shft, num_dims, i;
3907 kmp_int64 iter_number;
3908 kmp_info_t *th = __kmp_threads[gtid];
3909 kmp_team_t *team = th->th.th_team;
3911 kmp_int64 lo, up, st;
3913 KA_TRACE(20, (
"__kmpc_doacross_wait() enter: called T#%d\n", gtid));
3914 if (team->t.t_serialized) {
3915 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: serialized team\n"));
3920 pr_buf = th->th.th_dispatch;
3921 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3922 num_dims = pr_buf->th_doacross_info[0];
3923 lo = pr_buf->th_doacross_info[2];
3924 up = pr_buf->th_doacross_info[3];
3925 st = pr_buf->th_doacross_info[4];
3927 if (vec[0] < lo || vec[0] > up) {
3928 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3929 "bounds [%lld,%lld]\n",
3930 gtid, vec[0], lo, up));
3933 iter_number = vec[0] - lo;
3934 }
else if (st > 0) {
3935 if (vec[0] < lo || vec[0] > up) {
3936 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3937 "bounds [%lld,%lld]\n",
3938 gtid, vec[0], lo, up));
3941 iter_number = (kmp_uint64)(vec[0] - lo) / st;
3943 if (vec[0] > lo || vec[0] < up) {
3944 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3945 "bounds [%lld,%lld]\n",
3946 gtid, vec[0], lo, up));
3949 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
3951 for (i = 1; i < num_dims; ++i) {
3953 kmp_int32 j = i * 4;
3954 ln = pr_buf->th_doacross_info[j + 1];
3955 lo = pr_buf->th_doacross_info[j + 2];
3956 up = pr_buf->th_doacross_info[j + 3];
3957 st = pr_buf->th_doacross_info[j + 4];
3959 if (vec[i] < lo || vec[i] > up) {
3960 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3961 "bounds [%lld,%lld]\n",
3962 gtid, vec[i], lo, up));
3966 }
else if (st > 0) {
3967 if (vec[i] < lo || vec[i] > up) {
3968 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3969 "bounds [%lld,%lld]\n",
3970 gtid, vec[i], lo, up));
3973 iter = (kmp_uint64)(vec[i] - lo) / st;
3975 if (vec[i] > lo || vec[i] < up) {
3976 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3977 "bounds [%lld,%lld]\n",
3978 gtid, vec[i], lo, up));
3981 iter = (kmp_uint64)(lo - vec[i]) / (-st);
3983 iter_number = iter + ln * iter_number;
3985 shft = iter_number % 32;
3988 while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
3993 (
"__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
3994 gtid, (iter_number << 5) + shft));
3997 void __kmpc_doacross_post(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
3998 kmp_int32 shft, num_dims, i;
4000 kmp_int64 iter_number;
4001 kmp_info_t *th = __kmp_threads[gtid];
4002 kmp_team_t *team = th->th.th_team;
4006 KA_TRACE(20, (
"__kmpc_doacross_post() enter: called T#%d\n", gtid));
4007 if (team->t.t_serialized) {
4008 KA_TRACE(20, (
"__kmpc_doacross_post() exit: serialized team\n"));
4014 pr_buf = th->th.th_dispatch;
4015 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4016 num_dims = pr_buf->th_doacross_info[0];
4017 lo = pr_buf->th_doacross_info[2];
4018 st = pr_buf->th_doacross_info[4];
4020 iter_number = vec[0] - lo;
4021 }
else if (st > 0) {
4022 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4024 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4026 for (i = 1; i < num_dims; ++i) {
4028 kmp_int32 j = i * 4;
4029 ln = pr_buf->th_doacross_info[j + 1];
4030 lo = pr_buf->th_doacross_info[j + 2];
4031 st = pr_buf->th_doacross_info[j + 4];
4034 }
else if (st > 0) {
4035 iter = (kmp_uint64)(vec[i] - lo) / st;
4037 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4039 iter_number = iter + ln * iter_number;
4041 shft = iter_number % 32;
4045 if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
4046 KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
4047 KA_TRACE(20, (
"__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4048 (iter_number << 5) + shft));
4051 void __kmpc_doacross_fini(
ident_t *loc,
int gtid) {
4053 kmp_info_t *th = __kmp_threads[gtid];
4054 kmp_team_t *team = th->th.th_team;
4055 kmp_disp_t *pr_buf = th->th.th_dispatch;
4057 KA_TRACE(20, (
"__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4058 if (team->t.t_serialized) {
4059 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: serialized team %p\n", team));
4062 num_done = KMP_TEST_THEN_INC32((kmp_int32 *)pr_buf->th_doacross_info[1]) + 1;
4063 if (num_done == th->th.th_team_nproc) {
4065 int idx = pr_buf->th_doacross_buf_idx - 1;
4066 dispatch_shared_info_t *sh_buf =
4067 &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4068 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] ==
4069 (kmp_int64)&sh_buf->doacross_num_done);
4070 KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done);
4071 KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
4072 __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags));
4073 sh_buf->doacross_flags = NULL;
4074 sh_buf->doacross_num_done = 0;
4075 sh_buf->doacross_buf_idx +=
4076 __kmp_dispatch_num_buffers;
4079 pr_buf->th_doacross_flags = NULL;
4080 __kmp_thread_free(th, (
void *)pr_buf->th_doacross_info);
4081 pr_buf->th_doacross_info = NULL;
4082 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: T#%d\n", gtid));
4087 int __kmpc_get_target_offload(
void) {
4088 if (!__kmp_init_serial) {
4089 __kmp_serial_initialize();
4091 return __kmp_target_offload;
4093 #endif // OMP_50_ENABLED kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
#define KMP_IDENT_AUTOPAR
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
stats_state_e
the states which a thread can be in
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)