17 #include "kmp_error.h" 21 #include "kmp_stats.h" 24 #include "ompt-specific.h" 27 #define MAX_MESSAGE 512 43 if ((env = getenv(
"KMP_INITIAL_THREAD_BIND")) != NULL &&
44 __kmp_str_match_true(env)) {
45 __kmp_middle_initialize();
46 KC_TRACE(10, (
"__kmpc_begin: middle initialization called\n"));
47 }
else if (__kmp_ignore_mppbeg() == FALSE) {
49 __kmp_internal_begin();
50 KC_TRACE(10, (
"__kmpc_begin: called\n"));
68 if (__kmp_ignore_mppend() == FALSE) {
69 KC_TRACE(10, (
"__kmpc_end: called\n"));
70 KA_TRACE(30, (
"__kmpc_end\n"));
72 __kmp_internal_end_thread(-1);
74 #if KMP_OS_WINDOWS && OMPT_SUPPORT 79 if (ompt_enabled.enabled)
80 __kmp_internal_end_library(__kmp_gtid_get_specific());
103 kmp_int32 gtid = __kmp_entry_gtid();
105 KC_TRACE(10, (
"__kmpc_global_thread_num: T#%d\n", gtid));
126 (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
128 return TCR_4(__kmp_all_nth);
138 KC_TRACE(10, (
"__kmpc_bound_thread_num: called\n"));
139 return __kmp_tid_from_gtid(__kmp_entry_gtid());
148 KC_TRACE(10, (
"__kmpc_bound_num_threads: called\n"));
150 return __kmp_entry_thread()->th.th_team->t.t_nproc;
170 if (__kmp_par_range == 0) {
177 semi2 = strchr(semi2,
';');
181 semi2 = strchr(semi2 + 1,
';');
185 if (__kmp_par_range_filename[0]) {
186 const char *name = semi2 - 1;
187 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
190 if ((*name ==
'/') || (*name ==
';')) {
193 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
194 return __kmp_par_range < 0;
197 semi3 = strchr(semi2 + 1,
';');
198 if (__kmp_par_range_routine[0]) {
199 if ((semi3 != NULL) && (semi3 > semi2) &&
200 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
201 return __kmp_par_range < 0;
204 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
205 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
206 return __kmp_par_range > 0;
208 return __kmp_par_range < 0;
222 return __kmp_entry_thread()->th.th_root->r.r_active;
235 kmp_int32 num_threads) {
236 KA_TRACE(20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
237 global_tid, num_threads));
239 __kmp_push_num_threads(loc, global_tid, num_threads);
242 void __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid) {
243 KA_TRACE(20, (
"__kmpc_pop_num_threads: enter\n"));
250 void __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid,
251 kmp_int32 proc_bind) {
252 KA_TRACE(20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
255 __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
271 int gtid = __kmp_entry_gtid();
273 #if (KMP_STATS_ENABLED) 277 if (previous_state == stats_state_e::SERIAL_REGION) {
278 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
280 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead);
293 va_start(ap, microtask);
296 ompt_frame_t *ompt_frame;
297 if (ompt_enabled.enabled) {
298 kmp_info_t *master_th = __kmp_threads[gtid];
299 kmp_team_t *parent_team = master_th->th.th_team;
300 ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
302 ompt_frame = &(lwt->ompt_task_info.frame);
304 int tid = __kmp_tid_from_gtid(gtid);
306 parent_team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame);
308 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
309 OMPT_STORE_RETURN_ADDRESS(gtid);
313 #if INCLUDE_SSC_MARKS 316 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
317 VOLATILE_CAST(microtask_t) microtask,
318 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
320 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
326 #if INCLUDE_SSC_MARKS 329 __kmp_join_call(loc, gtid
339 #if KMP_STATS_ENABLED 340 if (previous_state == stats_state_e::SERIAL_REGION) {
341 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
343 KMP_POP_PARTITIONED_TIMER();
345 #endif // KMP_STATS_ENABLED 361 kmp_int32 num_teams, kmp_int32 num_threads) {
363 (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
364 global_tid, num_teams, num_threads));
366 __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
381 int gtid = __kmp_entry_gtid();
382 kmp_info_t *this_thr = __kmp_threads[gtid];
384 va_start(ap, microtask);
389 this_thr->th.th_teams_microtask = microtask;
390 this_thr->th.th_teams_level =
391 this_thr->th.th_team->t.t_level;
394 kmp_team_t *parent_team = this_thr->th.th_team;
395 int tid = __kmp_tid_from_gtid(gtid);
396 if (ompt_enabled.enabled) {
397 parent_team->t.t_implicit_task_taskdata[tid]
398 .ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
400 OMPT_STORE_RETURN_ADDRESS(gtid);
405 if (this_thr->th.th_teams_size.nteams == 0) {
406 __kmp_push_num_teams(loc, gtid, 0, 0);
408 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
409 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
410 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
412 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
413 VOLATILE_CAST(microtask_t)
415 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
416 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
422 __kmp_join_call(loc, gtid
429 this_thr->th.th_teams_microtask = NULL;
430 this_thr->th.th_teams_level = 0;
431 *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
440 int __kmpc_invoke_task_func(
int gtid) {
return __kmp_invoke_task_func(gtid); }
459 OMPT_STORE_RETURN_ADDRESS(global_tid);
461 __kmp_serialized_parallel(loc, global_tid);
472 kmp_internal_control_t *top;
473 kmp_info_t *this_thr;
474 kmp_team_t *serial_team;
477 (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
485 if (!TCR_4(__kmp_init_parallel))
486 __kmp_parallel_initialize();
488 this_thr = __kmp_threads[global_tid];
489 serial_team = this_thr->th.th_serial_team;
492 kmp_task_team_t *task_team = this_thr->th.th_task_team;
495 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)
496 __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
500 KMP_DEBUG_ASSERT(serial_team);
501 KMP_ASSERT(serial_team->t.t_serialized);
502 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
503 KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
504 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
505 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
508 if (ompt_enabled.enabled &&
509 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
510 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = ompt_data_none;
511 if (ompt_enabled.ompt_callback_implicit_task) {
512 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
513 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
514 OMPT_CUR_TASK_INFO(this_thr)->thread_num, ompt_task_implicit);
518 ompt_data_t *parent_task_data;
519 __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
521 if (ompt_enabled.ompt_callback_parallel_end) {
522 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
523 &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
524 ompt_parallel_invoker_program, OMPT_LOAD_RETURN_ADDRESS(global_tid));
526 __ompt_lw_taskteam_unlink(this_thr);
527 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
533 top = serial_team->t.t_control_stack_top;
534 if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
535 copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
536 serial_team->t.t_control_stack_top = top->next;
541 serial_team->t.t_level--;
544 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
546 dispatch_private_info_t *disp_buffer =
547 serial_team->t.t_dispatch->th_disp_buffer;
548 serial_team->t.t_dispatch->th_disp_buffer =
549 serial_team->t.t_dispatch->th_disp_buffer->next;
550 __kmp_free(disp_buffer);
553 this_thr->th.th_def_allocator = serial_team->t.t_def_allocator;
556 --serial_team->t.t_serialized;
557 if (serial_team->t.t_serialized == 0) {
561 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 562 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
563 __kmp_clear_x87_fpu_status_word();
564 __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
565 __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
569 this_thr->th.th_team = serial_team->t.t_parent;
570 this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
573 this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc;
574 this_thr->th.th_team_master =
575 serial_team->t.t_parent->t.t_threads[0];
576 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
579 this_thr->th.th_dispatch =
580 &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
582 __kmp_pop_current_task_from_thread(this_thr);
584 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
585 this_thr->th.th_current_task->td_flags.executing = 1;
587 if (__kmp_tasking_mode != tskm_immediate_exec) {
589 this_thr->th.th_task_team =
590 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
592 (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / " 594 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
597 if (__kmp_tasking_mode != tskm_immediate_exec) {
598 KA_TRACE(20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting " 599 "depth of serial team %p to %d\n",
600 global_tid, serial_team, serial_team->t.t_serialized));
604 if (__kmp_env_consistency_check)
605 __kmp_pop_parallel(global_tid, NULL);
607 if (ompt_enabled.enabled)
608 this_thr->th.ompt_thread_info.state =
609 ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
610 : ompt_state_work_parallel);
623 KC_TRACE(10, (
"__kmpc_flush: called\n"));
628 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 642 if (!__kmp_cpuinfo.initialized) {
643 __kmp_query_cpuid(&__kmp_cpuinfo);
645 if (!__kmp_cpuinfo.sse2) {
650 #elif KMP_COMPILER_MSVC 653 __sync_synchronize();
654 #endif // KMP_COMPILER_ICC 657 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64) 673 #error Unknown or unsupported architecture 676 #if OMPT_SUPPORT && OMPT_OPTIONAL 677 if (ompt_enabled.ompt_callback_flush) {
678 ompt_callbacks.ompt_callback(ompt_callback_flush)(
679 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
694 KC_TRACE(10, (
"__kmpc_barrier: called T#%d\n", global_tid));
696 if (!TCR_4(__kmp_init_parallel))
697 __kmp_parallel_initialize();
699 if (__kmp_env_consistency_check) {
701 KMP_WARNING(ConstructIdentInvalid);
704 __kmp_check_barrier(global_tid, ct_barrier, loc);
708 ompt_frame_t *ompt_frame;
709 if (ompt_enabled.enabled) {
710 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
711 if (ompt_frame->enter_frame.ptr == NULL)
712 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
713 OMPT_STORE_RETURN_ADDRESS(global_tid);
716 __kmp_threads[global_tid]->th.th_ident = loc;
724 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
725 #if OMPT_SUPPORT && OMPT_OPTIONAL 726 if (ompt_enabled.enabled) {
727 ompt_frame->enter_frame = ompt_data_none;
742 KC_TRACE(10, (
"__kmpc_master: called T#%d\n", global_tid));
744 if (!TCR_4(__kmp_init_parallel))
745 __kmp_parallel_initialize();
747 if (KMP_MASTER_GTID(global_tid)) {
749 KMP_PUSH_PARTITIONED_TIMER(OMP_master);
753 #if OMPT_SUPPORT && OMPT_OPTIONAL 755 if (ompt_enabled.ompt_callback_master) {
756 kmp_info_t *this_thr = __kmp_threads[global_tid];
757 kmp_team_t *team = this_thr->th.th_team;
759 int tid = __kmp_tid_from_gtid(global_tid);
760 ompt_callbacks.ompt_callback(ompt_callback_master)(
761 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
762 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
763 OMPT_GET_RETURN_ADDRESS(0));
768 if (__kmp_env_consistency_check) {
769 #if KMP_USE_DYNAMIC_LOCK 771 __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
773 __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
776 __kmp_push_sync(global_tid, ct_master, loc, NULL);
778 __kmp_check_sync(global_tid, ct_master, loc, NULL);
794 KC_TRACE(10, (
"__kmpc_end_master: called T#%d\n", global_tid));
796 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
797 KMP_POP_PARTITIONED_TIMER();
799 #if OMPT_SUPPORT && OMPT_OPTIONAL 800 kmp_info_t *this_thr = __kmp_threads[global_tid];
801 kmp_team_t *team = this_thr->th.th_team;
802 if (ompt_enabled.ompt_callback_master) {
803 int tid = __kmp_tid_from_gtid(global_tid);
804 ompt_callbacks.ompt_callback(ompt_callback_master)(
805 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
806 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
807 OMPT_GET_RETURN_ADDRESS(0));
811 if (__kmp_env_consistency_check) {
813 KMP_WARNING(ThreadIdentInvalid);
815 if (KMP_MASTER_GTID(global_tid))
816 __kmp_pop_sync(global_tid, ct_master, loc);
830 KMP_DEBUG_ASSERT(__kmp_init_serial);
832 KC_TRACE(10, (
"__kmpc_ordered: called T#%d\n", gtid));
834 if (!TCR_4(__kmp_init_parallel))
835 __kmp_parallel_initialize();
838 __kmp_itt_ordered_prep(gtid);
842 th = __kmp_threads[gtid];
844 #if OMPT_SUPPORT && OMPT_OPTIONAL 848 if (ompt_enabled.enabled) {
849 OMPT_STORE_RETURN_ADDRESS(gtid);
850 team = __kmp_team_from_gtid(gtid);
851 lck = (ompt_wait_id_t)&team->t.t_ordered.dt.t_value;
853 th->th.ompt_thread_info.wait_id = lck;
854 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
857 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
858 if (ompt_enabled.ompt_callback_mutex_acquire) {
859 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
860 ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin,
861 (ompt_wait_id_t)lck, codeptr_ra);
866 if (th->th.th_dispatch->th_deo_fcn != 0)
867 (*th->th.th_dispatch->th_deo_fcn)(>id, &cid, loc);
869 __kmp_parallel_deo(>id, &cid, loc);
871 #if OMPT_SUPPORT && OMPT_OPTIONAL 872 if (ompt_enabled.enabled) {
874 th->th.ompt_thread_info.state = ompt_state_work_parallel;
875 th->th.ompt_thread_info.wait_id = 0;
878 if (ompt_enabled.ompt_callback_mutex_acquired) {
879 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
880 ompt_mutex_ordered, (ompt_wait_id_t)lck, codeptr_ra);
886 __kmp_itt_ordered_start(gtid);
901 KC_TRACE(10, (
"__kmpc_end_ordered: called T#%d\n", gtid));
904 __kmp_itt_ordered_end(gtid);
908 th = __kmp_threads[gtid];
910 if (th->th.th_dispatch->th_dxo_fcn != 0)
911 (*th->th.th_dispatch->th_dxo_fcn)(>id, &cid, loc);
913 __kmp_parallel_dxo(>id, &cid, loc);
915 #if OMPT_SUPPORT && OMPT_OPTIONAL 916 OMPT_STORE_RETURN_ADDRESS(gtid);
917 if (ompt_enabled.ompt_callback_mutex_released) {
918 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
920 (ompt_wait_id_t)&__kmp_team_from_gtid(gtid)->t.t_ordered.dt.t_value,
921 OMPT_LOAD_RETURN_ADDRESS(gtid));
926 #if KMP_USE_DYNAMIC_LOCK 928 static __forceinline
void 929 __kmp_init_indirect_csptr(kmp_critical_name *crit,
ident_t const *loc,
930 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
934 kmp_indirect_lock_t **lck;
935 lck = (kmp_indirect_lock_t **)crit;
936 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
937 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
938 KMP_SET_I_LOCK_LOCATION(ilk, loc);
939 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
941 (
"__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
943 __kmp_itt_critical_creating(ilk->lock, loc);
945 int status = KMP_COMPARE_AND_STORE_PTR(lck,
nullptr, ilk);
948 __kmp_itt_critical_destroyed(ilk->lock);
954 KMP_DEBUG_ASSERT(*lck != NULL);
958 #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \ 960 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 961 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 962 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 963 if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 964 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 966 KMP_FSYNC_PREPARE(l); \ 967 KMP_INIT_YIELD(spins); \ 968 if (TCR_4(__kmp_nth) > \ 969 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 972 KMP_YIELD_SPIN(spins); \ 974 kmp_backoff_t backoff = __kmp_spin_backoff_params; \ 976 KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \ 977 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \ 978 __kmp_spin_backoff(&backoff); \ 979 if (TCR_4(__kmp_nth) > \ 980 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 983 KMP_YIELD_SPIN(spins); \ 987 KMP_FSYNC_ACQUIRED(l); \ 991 #define KMP_TEST_TAS_LOCK(lock, gtid, rc) \ 993 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 994 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \ 995 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \ 996 rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \ 997 __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \ 1001 #define KMP_RELEASE_TAS_LOCK(lock, gtid) \ 1002 { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); } 1006 #include <sys/syscall.h> 1009 #define FUTEX_WAIT 0 1012 #define FUTEX_WAKE 1 1016 #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \ 1018 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1019 kmp_int32 gtid_code = (gtid + 1) << 1; \ 1021 KMP_FSYNC_PREPARE(ftx); \ 1022 kmp_int32 poll_val; \ 1023 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \ 1024 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1025 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ 1026 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ 1028 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \ 1030 KMP_LOCK_BUSY(1, futex))) { \ 1033 poll_val |= KMP_LOCK_BUSY(1, futex); \ 1036 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \ 1037 NULL, NULL, 0)) != 0) { \ 1042 KMP_FSYNC_ACQUIRED(ftx); \ 1046 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \ 1048 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1049 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1050 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \ 1051 KMP_FSYNC_ACQUIRED(ftx); \ 1059 #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \ 1061 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1063 KMP_FSYNC_RELEASING(ftx); \ 1064 kmp_int32 poll_val = \ 1065 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ 1066 if (KMP_LOCK_STRIP(poll_val) & 1) { \ 1067 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \ 1068 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ 1071 KMP_YIELD(TCR_4(__kmp_nth) > \ 1072 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ 1075 #endif // KMP_USE_FUTEX 1077 #else // KMP_USE_DYNAMIC_LOCK 1079 static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1082 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1085 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1092 lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1093 __kmp_init_user_lock_with_checks(lck);
1094 __kmp_set_user_lock_location(lck, loc);
1096 __kmp_itt_critical_creating(lck);
1107 int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1112 __kmp_itt_critical_destroyed(lck);
1116 __kmp_destroy_user_lock_with_checks(lck);
1117 __kmp_user_lock_free(&idx, gtid, lck);
1118 lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1119 KMP_DEBUG_ASSERT(lck != NULL);
1125 #endif // KMP_USE_DYNAMIC_LOCK 1138 kmp_critical_name *crit) {
1139 #if KMP_USE_DYNAMIC_LOCK 1140 #if OMPT_SUPPORT && OMPT_OPTIONAL 1141 OMPT_STORE_RETURN_ADDRESS(global_tid);
1142 #endif // OMPT_SUPPORT 1143 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1146 #if OMPT_SUPPORT && OMPT_OPTIONAL 1147 ompt_state_t prev_state = ompt_state_undefined;
1148 ompt_thread_info_t ti;
1150 kmp_user_lock_p lck;
1152 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1156 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1157 KMP_CHECK_USER_LOCK_INIT();
1159 if ((__kmp_user_lock_kind == lk_tas) &&
1160 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1161 lck = (kmp_user_lock_p)crit;
1164 else if ((__kmp_user_lock_kind == lk_futex) &&
1165 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1166 lck = (kmp_user_lock_p)crit;
1170 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1173 if (__kmp_env_consistency_check)
1174 __kmp_push_sync(global_tid, ct_critical, loc, lck);
1182 __kmp_itt_critical_acquiring(lck);
1184 #if OMPT_SUPPORT && OMPT_OPTIONAL 1185 OMPT_STORE_RETURN_ADDRESS(gtid);
1186 void *codeptr_ra = NULL;
1187 if (ompt_enabled.enabled) {
1188 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1190 prev_state = ti.state;
1191 ti.wait_id = (ompt_wait_id_t)lck;
1192 ti.state = ompt_state_wait_critical;
1195 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1196 if (ompt_enabled.ompt_callback_mutex_acquire) {
1197 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1198 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1199 (ompt_wait_id_t)crit, codeptr_ra);
1205 __kmp_acquire_user_lock_with_checks(lck, global_tid);
1208 __kmp_itt_critical_acquired(lck);
1210 #if OMPT_SUPPORT && OMPT_OPTIONAL 1211 if (ompt_enabled.enabled) {
1213 ti.state = prev_state;
1217 if (ompt_enabled.ompt_callback_mutex_acquired) {
1218 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1219 ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr_ra);
1223 KMP_POP_PARTITIONED_TIMER();
1225 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1226 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1227 #endif // KMP_USE_DYNAMIC_LOCK 1230 #if KMP_USE_DYNAMIC_LOCK 1233 static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1235 #define KMP_TSX_LOCK(seq) lockseq_##seq 1237 #define KMP_TSX_LOCK(seq) __kmp_user_lock_seq 1240 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1241 #define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm) 1243 #define KMP_CPUINFO_RTM 0 1247 if (hint & kmp_lock_hint_hle)
1248 return KMP_TSX_LOCK(hle);
1249 if (hint & kmp_lock_hint_rtm)
1250 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm) : __kmp_user_lock_seq;
1251 if (hint & kmp_lock_hint_adaptive)
1252 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1255 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1256 return __kmp_user_lock_seq;
1257 if ((hint & omp_lock_hint_speculative) &&
1258 (hint & omp_lock_hint_nonspeculative))
1259 return __kmp_user_lock_seq;
1262 if (hint & omp_lock_hint_contended)
1263 return lockseq_queuing;
1266 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1270 if (hint & omp_lock_hint_speculative)
1271 return KMP_TSX_LOCK(hle);
1273 return __kmp_user_lock_seq;
1276 #if OMPT_SUPPORT && OMPT_OPTIONAL 1277 #if KMP_USE_DYNAMIC_LOCK 1278 static kmp_mutex_impl_t
1279 __ompt_get_mutex_impl_type(
void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1281 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1286 return kmp_mutex_impl_queuing;
1289 return kmp_mutex_impl_spin;
1292 return kmp_mutex_impl_speculative;
1295 return kmp_mutex_impl_none;
1297 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1300 switch (ilock->type) {
1302 case locktag_adaptive:
1304 return kmp_mutex_impl_speculative;
1306 case locktag_nested_tas:
1307 return kmp_mutex_impl_spin;
1309 case locktag_nested_futex:
1311 case locktag_ticket:
1312 case locktag_queuing:
1314 case locktag_nested_ticket:
1315 case locktag_nested_queuing:
1316 case locktag_nested_drdpa:
1317 return kmp_mutex_impl_queuing;
1319 return kmp_mutex_impl_none;
1324 static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
1325 switch (__kmp_user_lock_kind) {
1327 return kmp_mutex_impl_spin;
1334 return kmp_mutex_impl_queuing;
1339 return kmp_mutex_impl_speculative;
1342 return kmp_mutex_impl_none;
1345 #endif // KMP_USE_DYNAMIC_LOCK 1346 #endif // OMPT_SUPPORT && OMPT_OPTIONAL 1361 void __kmpc_critical_with_hint(
ident_t *loc, kmp_int32 global_tid,
1362 kmp_critical_name *crit, uint32_t hint) {
1364 kmp_user_lock_p lck;
1365 #if OMPT_SUPPORT && OMPT_OPTIONAL 1366 ompt_state_t prev_state = ompt_state_undefined;
1367 ompt_thread_info_t ti;
1369 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1371 codeptr = OMPT_GET_RETURN_ADDRESS(0);
1374 KC_TRACE(10, (
"__kmpc_critical: called T#%d\n", global_tid));
1376 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1378 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1380 kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
1381 if (KMP_IS_D_LOCK(lckseq)) {
1382 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
1383 KMP_GET_D_TAG(lckseq));
1385 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
1391 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1392 lck = (kmp_user_lock_p)lk;
1393 if (__kmp_env_consistency_check) {
1394 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1395 __kmp_map_hint_to_lock(hint));
1398 __kmp_itt_critical_acquiring(lck);
1400 #if OMPT_SUPPORT && OMPT_OPTIONAL 1401 if (ompt_enabled.enabled) {
1402 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1404 prev_state = ti.state;
1405 ti.wait_id = (ompt_wait_id_t)lck;
1406 ti.state = ompt_state_wait_critical;
1409 if (ompt_enabled.ompt_callback_mutex_acquire) {
1410 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1411 ompt_mutex_critical, (
unsigned int)hint,
1412 __ompt_get_mutex_impl_type(crit), (ompt_wait_id_t)crit, codeptr);
1416 #if KMP_USE_INLINED_TAS 1417 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1418 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1420 #elif KMP_USE_INLINED_FUTEX 1421 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1422 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1426 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
1429 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1431 if (__kmp_env_consistency_check) {
1432 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1433 __kmp_map_hint_to_lock(hint));
1436 __kmp_itt_critical_acquiring(lck);
1438 #if OMPT_SUPPORT && OMPT_OPTIONAL 1439 if (ompt_enabled.enabled) {
1440 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1442 prev_state = ti.state;
1443 ti.wait_id = (ompt_wait_id_t)lck;
1444 ti.state = ompt_state_wait_critical;
1447 if (ompt_enabled.ompt_callback_mutex_acquire) {
1448 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1449 ompt_mutex_critical, (
unsigned int)hint,
1450 __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)crit, codeptr);
1454 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
1456 KMP_POP_PARTITIONED_TIMER();
1459 __kmp_itt_critical_acquired(lck);
1461 #if OMPT_SUPPORT && OMPT_OPTIONAL 1462 if (ompt_enabled.enabled) {
1464 ti.state = prev_state;
1468 if (ompt_enabled.ompt_callback_mutex_acquired) {
1469 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1470 ompt_mutex_critical, (ompt_wait_id_t)crit, codeptr);
1475 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1476 KA_TRACE(15, (
"__kmpc_critical: done T#%d\n", global_tid));
1479 #endif // KMP_USE_DYNAMIC_LOCK 1491 kmp_critical_name *crit) {
1492 kmp_user_lock_p lck;
1494 KC_TRACE(10, (
"__kmpc_end_critical: called T#%d\n", global_tid));
1496 #if KMP_USE_DYNAMIC_LOCK 1497 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
1498 lck = (kmp_user_lock_p)crit;
1499 KMP_ASSERT(lck != NULL);
1500 if (__kmp_env_consistency_check) {
1501 __kmp_pop_sync(global_tid, ct_critical, loc);
1504 __kmp_itt_critical_releasing(lck);
1506 #if KMP_USE_INLINED_TAS 1507 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1508 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1510 #elif KMP_USE_INLINED_FUTEX 1511 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1512 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1516 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1519 kmp_indirect_lock_t *ilk =
1520 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1521 KMP_ASSERT(ilk != NULL);
1523 if (__kmp_env_consistency_check) {
1524 __kmp_pop_sync(global_tid, ct_critical, loc);
1527 __kmp_itt_critical_releasing(lck);
1529 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1532 #else // KMP_USE_DYNAMIC_LOCK 1534 if ((__kmp_user_lock_kind == lk_tas) &&
1535 (
sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1536 lck = (kmp_user_lock_p)crit;
1539 else if ((__kmp_user_lock_kind == lk_futex) &&
1540 (
sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1541 lck = (kmp_user_lock_p)crit;
1545 lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit));
1548 KMP_ASSERT(lck != NULL);
1550 if (__kmp_env_consistency_check)
1551 __kmp_pop_sync(global_tid, ct_critical, loc);
1554 __kmp_itt_critical_releasing(lck);
1558 __kmp_release_user_lock_with_checks(lck, global_tid);
1560 #endif // KMP_USE_DYNAMIC_LOCK 1562 #if OMPT_SUPPORT && OMPT_OPTIONAL 1565 OMPT_STORE_RETURN_ADDRESS(global_tid);
1566 if (ompt_enabled.ompt_callback_mutex_released) {
1567 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1568 ompt_mutex_critical, (ompt_wait_id_t)crit, OMPT_LOAD_RETURN_ADDRESS(0));
1572 KMP_POP_PARTITIONED_TIMER();
1573 KA_TRACE(15, (
"__kmpc_end_critical: done T#%d\n", global_tid));
1588 KC_TRACE(10, (
"__kmpc_barrier_master: called T#%d\n", global_tid));
1590 if (!TCR_4(__kmp_init_parallel))
1591 __kmp_parallel_initialize();
1593 if (__kmp_env_consistency_check)
1594 __kmp_check_barrier(global_tid, ct_barrier, loc);
1597 ompt_frame_t *ompt_frame;
1598 if (ompt_enabled.enabled) {
1599 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1600 if (ompt_frame->enter_frame.ptr == NULL)
1601 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1602 OMPT_STORE_RETURN_ADDRESS(global_tid);
1606 __kmp_threads[global_tid]->th.th_ident = loc;
1608 status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL);
1609 #if OMPT_SUPPORT && OMPT_OPTIONAL 1610 if (ompt_enabled.enabled) {
1611 ompt_frame->enter_frame = ompt_data_none;
1615 return (status != 0) ? 0 : 1;
1628 KC_TRACE(10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid));
1630 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1646 KC_TRACE(10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1648 if (!TCR_4(__kmp_init_parallel))
1649 __kmp_parallel_initialize();
1651 if (__kmp_env_consistency_check) {
1653 KMP_WARNING(ConstructIdentInvalid);
1655 __kmp_check_barrier(global_tid, ct_barrier, loc);
1659 ompt_frame_t *ompt_frame;
1660 if (ompt_enabled.enabled) {
1661 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1662 if (ompt_frame->enter_frame.ptr == NULL)
1663 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1664 OMPT_STORE_RETURN_ADDRESS(global_tid);
1668 __kmp_threads[global_tid]->th.th_ident = loc;
1670 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
1671 #if OMPT_SUPPORT && OMPT_OPTIONAL 1672 if (ompt_enabled.enabled) {
1673 ompt_frame->enter_frame = ompt_data_none;
1679 if (__kmp_env_consistency_check) {
1683 if (global_tid < 0) {
1684 KMP_WARNING(ThreadIdentInvalid);
1690 __kmp_pop_sync(global_tid, ct_master, loc);
1710 kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
1715 KMP_PUSH_PARTITIONED_TIMER(OMP_single);
1718 #if OMPT_SUPPORT && OMPT_OPTIONAL 1719 kmp_info_t *this_thr = __kmp_threads[global_tid];
1720 kmp_team_t *team = this_thr->th.th_team;
1721 int tid = __kmp_tid_from_gtid(global_tid);
1723 if (ompt_enabled.enabled) {
1725 if (ompt_enabled.ompt_callback_work) {
1726 ompt_callbacks.ompt_callback(ompt_callback_work)(
1727 ompt_work_single_executor, ompt_scope_begin,
1728 &(team->t.ompt_team_info.parallel_data),
1729 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1730 1, OMPT_GET_RETURN_ADDRESS(0));
1733 if (ompt_enabled.ompt_callback_work) {
1734 ompt_callbacks.ompt_callback(ompt_callback_work)(
1735 ompt_work_single_other, ompt_scope_begin,
1736 &(team->t.ompt_team_info.parallel_data),
1737 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1738 1, OMPT_GET_RETURN_ADDRESS(0));
1739 ompt_callbacks.ompt_callback(ompt_callback_work)(
1740 ompt_work_single_other, ompt_scope_end,
1741 &(team->t.ompt_team_info.parallel_data),
1742 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1743 1, OMPT_GET_RETURN_ADDRESS(0));
1762 __kmp_exit_single(global_tid);
1763 KMP_POP_PARTITIONED_TIMER();
1765 #if OMPT_SUPPORT && OMPT_OPTIONAL 1766 kmp_info_t *this_thr = __kmp_threads[global_tid];
1767 kmp_team_t *team = this_thr->th.th_team;
1768 int tid = __kmp_tid_from_gtid(global_tid);
1770 if (ompt_enabled.ompt_callback_work) {
1771 ompt_callbacks.ompt_callback(ompt_callback_work)(
1772 ompt_work_single_executor, ompt_scope_end,
1773 &(team->t.ompt_team_info.parallel_data),
1774 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
1775 OMPT_GET_RETURN_ADDRESS(0));
1788 KMP_POP_PARTITIONED_TIMER();
1789 KE_TRACE(10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1791 #if OMPT_SUPPORT && OMPT_OPTIONAL 1792 if (ompt_enabled.ompt_callback_work) {
1793 ompt_work_t ompt_work_type = ompt_work_loop;
1794 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
1795 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
1799 ompt_work_type = ompt_work_loop;
1801 ompt_work_type = ompt_work_sections;
1803 ompt_work_type = ompt_work_distribute;
1808 KMP_DEBUG_ASSERT(ompt_work_type);
1810 ompt_callbacks.ompt_callback(ompt_callback_work)(
1811 ompt_work_type, ompt_scope_end, &(team_info->parallel_data),
1812 &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
1815 if (__kmp_env_consistency_check)
1816 __kmp_pop_workshare(global_tid, ct_pdo, loc);
1822 void ompc_set_num_threads(
int arg) {
1824 __kmp_set_num_threads(arg, __kmp_entry_gtid());
1827 void ompc_set_dynamic(
int flag) {
1831 thread = __kmp_entry_thread();
1833 __kmp_save_internal_controls(thread);
1835 set__dynamic(thread, flag ? TRUE : FALSE);
1838 void ompc_set_nested(
int flag) {
1842 thread = __kmp_entry_thread();
1844 __kmp_save_internal_controls(thread);
1846 set__nested(thread, flag ? TRUE : FALSE);
1849 void ompc_set_max_active_levels(
int max_active_levels) {
1854 __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);
1857 void ompc_set_schedule(omp_sched_t kind,
int modifier) {
1859 __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier);
1862 int ompc_get_ancestor_thread_num(
int level) {
1863 return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level);
1866 int ompc_get_team_size(
int level) {
1867 return __kmp_get_team_size(__kmp_entry_gtid(), level);
1873 void ompc_set_affinity_format(
char const *format) {
1874 if (!__kmp_init_serial) {
1875 __kmp_serial_initialize();
1877 __kmp_strncpy_truncate(__kmp_affinity_format, KMP_AFFINITY_FORMAT_SIZE,
1878 format, KMP_STRLEN(format) + 1);
1881 size_t ompc_get_affinity_format(
char *buffer,
size_t size) {
1883 if (!__kmp_init_serial) {
1884 __kmp_serial_initialize();
1886 format_size = KMP_STRLEN(__kmp_affinity_format);
1887 if (buffer && size) {
1888 __kmp_strncpy_truncate(buffer, size, __kmp_affinity_format,
1894 void ompc_display_affinity(
char const *format) {
1896 if (!TCR_4(__kmp_init_middle)) {
1897 __kmp_middle_initialize();
1899 gtid = __kmp_get_gtid();
1900 __kmp_aux_display_affinity(gtid, format);
1903 size_t ompc_capture_affinity(
char *buffer,
size_t buf_size,
1904 char const *format) {
1906 size_t num_required;
1907 kmp_str_buf_t capture_buf;
1908 if (!TCR_4(__kmp_init_middle)) {
1909 __kmp_middle_initialize();
1911 gtid = __kmp_get_gtid();
1912 __kmp_str_buf_init(&capture_buf);
1913 num_required = __kmp_aux_capture_affinity(gtid, format, &capture_buf);
1914 if (buffer && buf_size) {
1915 __kmp_strncpy_truncate(buffer, buf_size, capture_buf.str,
1916 capture_buf.used + 1);
1918 __kmp_str_buf_free(&capture_buf);
1919 return num_required;
1923 void kmpc_set_stacksize(
int arg) {
1925 __kmp_aux_set_stacksize(arg);
1928 void kmpc_set_stacksize_s(
size_t arg) {
1930 __kmp_aux_set_stacksize(arg);
1933 void kmpc_set_blocktime(
int arg) {
1937 gtid = __kmp_entry_gtid();
1938 tid = __kmp_tid_from_gtid(gtid);
1939 thread = __kmp_thread_from_gtid(gtid);
1941 __kmp_aux_set_blocktime(arg, thread, tid);
1944 void kmpc_set_library(
int arg) {
1946 __kmp_user_set_library((
enum library_type)arg);
1949 void kmpc_set_defaults(
char const *str) {
1951 __kmp_aux_set_defaults(str, KMP_STRLEN(str));
1954 void kmpc_set_disp_num_buffers(
int arg) {
1957 if (__kmp_init_serial == 0 && arg > 0)
1958 __kmp_dispatch_num_buffers = arg;
1961 int kmpc_set_affinity_mask_proc(
int proc,
void **mask) {
1962 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1965 if (!TCR_4(__kmp_init_middle)) {
1966 __kmp_middle_initialize();
1968 return __kmp_aux_set_affinity_mask_proc(proc, mask);
1972 int kmpc_unset_affinity_mask_proc(
int proc,
void **mask) {
1973 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1976 if (!TCR_4(__kmp_init_middle)) {
1977 __kmp_middle_initialize();
1979 return __kmp_aux_unset_affinity_mask_proc(proc, mask);
1983 int kmpc_get_affinity_mask_proc(
int proc,
void **mask) {
1984 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1987 if (!TCR_4(__kmp_init_middle)) {
1988 __kmp_middle_initialize();
1990 return __kmp_aux_get_affinity_mask_proc(proc, mask);
2040 void *cpy_data,
void (*cpy_func)(
void *,
void *),
2044 KC_TRACE(10, (
"__kmpc_copyprivate: called T#%d\n", gtid));
2048 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
2050 if (__kmp_env_consistency_check) {
2052 KMP_WARNING(ConstructIdentInvalid);
2059 *data_ptr = cpy_data;
2062 ompt_frame_t *ompt_frame;
2063 if (ompt_enabled.enabled) {
2064 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
2065 if (ompt_frame->enter_frame.ptr == NULL)
2066 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
2067 OMPT_STORE_RETURN_ADDRESS(gtid);
2072 __kmp_threads[gtid]->th.th_ident = loc;
2074 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2077 (*cpy_func)(cpy_data, *data_ptr);
2083 if (ompt_enabled.enabled) {
2084 OMPT_STORE_RETURN_ADDRESS(gtid);
2088 __kmp_threads[gtid]->th.th_ident = loc;
2091 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2092 #if OMPT_SUPPORT && OMPT_OPTIONAL 2093 if (ompt_enabled.enabled) {
2094 ompt_frame->enter_frame = ompt_data_none;
2101 #define INIT_LOCK __kmp_init_user_lock_with_checks 2102 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks 2103 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks 2104 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed 2105 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks 2106 #define ACQUIRE_NESTED_LOCK_TIMED \ 2107 __kmp_acquire_nested_user_lock_with_checks_timed 2108 #define RELEASE_LOCK __kmp_release_user_lock_with_checks 2109 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks 2110 #define TEST_LOCK __kmp_test_user_lock_with_checks 2111 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks 2112 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks 2113 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks 2118 #if KMP_USE_DYNAMIC_LOCK 2121 static __forceinline
void __kmp_init_lock_with_hint(
ident_t *loc,
void **lock,
2122 kmp_dyna_lockseq_t seq) {
2123 if (KMP_IS_D_LOCK(seq)) {
2124 KMP_INIT_D_LOCK(lock, seq);
2126 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
2129 KMP_INIT_I_LOCK(lock, seq);
2131 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2132 __kmp_itt_lock_creating(ilk->lock, loc);
2138 static __forceinline
void 2139 __kmp_init_nest_lock_with_hint(
ident_t *loc,
void **lock,
2140 kmp_dyna_lockseq_t seq) {
2143 if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
2144 seq = __kmp_user_lock_seq;
2148 seq = lockseq_nested_tas;
2152 seq = lockseq_nested_futex;
2155 case lockseq_ticket:
2156 seq = lockseq_nested_ticket;
2158 case lockseq_queuing:
2159 seq = lockseq_nested_queuing;
2162 seq = lockseq_nested_drdpa;
2165 seq = lockseq_nested_queuing;
2167 KMP_INIT_I_LOCK(lock, seq);
2169 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2170 __kmp_itt_lock_creating(ilk->lock, loc);
2175 void __kmpc_init_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
void **user_lock,
2177 KMP_DEBUG_ASSERT(__kmp_init_serial);
2178 if (__kmp_env_consistency_check && user_lock == NULL) {
2179 KMP_FATAL(LockIsUninitialized,
"omp_init_lock_with_hint");
2182 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2184 #if OMPT_SUPPORT && OMPT_OPTIONAL 2186 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2188 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2189 if (ompt_enabled.ompt_callback_lock_init) {
2190 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2191 ompt_mutex_lock, (omp_lock_hint_t)hint,
2192 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2199 void __kmpc_init_nest_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
2200 void **user_lock, uintptr_t hint) {
2201 KMP_DEBUG_ASSERT(__kmp_init_serial);
2202 if (__kmp_env_consistency_check && user_lock == NULL) {
2203 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock_with_hint");
2206 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2208 #if OMPT_SUPPORT && OMPT_OPTIONAL 2210 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2212 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2213 if (ompt_enabled.ompt_callback_lock_init) {
2214 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2215 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2216 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2222 #endif // KMP_USE_DYNAMIC_LOCK 2225 void __kmpc_init_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2226 #if KMP_USE_DYNAMIC_LOCK 2228 KMP_DEBUG_ASSERT(__kmp_init_serial);
2229 if (__kmp_env_consistency_check && user_lock == NULL) {
2230 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
2232 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2234 #if OMPT_SUPPORT && OMPT_OPTIONAL 2236 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2238 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2239 if (ompt_enabled.ompt_callback_lock_init) {
2240 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2241 ompt_mutex_lock, omp_lock_hint_none,
2242 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2247 #else // KMP_USE_DYNAMIC_LOCK 2249 static char const *
const func =
"omp_init_lock";
2250 kmp_user_lock_p lck;
2251 KMP_DEBUG_ASSERT(__kmp_init_serial);
2253 if (__kmp_env_consistency_check) {
2254 if (user_lock == NULL) {
2255 KMP_FATAL(LockIsUninitialized, func);
2259 KMP_CHECK_USER_LOCK_INIT();
2261 if ((__kmp_user_lock_kind == lk_tas) &&
2262 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2263 lck = (kmp_user_lock_p)user_lock;
2266 else if ((__kmp_user_lock_kind == lk_futex) &&
2267 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2268 lck = (kmp_user_lock_p)user_lock;
2272 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2275 __kmp_set_user_lock_location(lck, loc);
2277 #if OMPT_SUPPORT && OMPT_OPTIONAL 2279 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2281 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2282 if (ompt_enabled.ompt_callback_lock_init) {
2283 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2284 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2285 (ompt_wait_id_t)user_lock, codeptr);
2290 __kmp_itt_lock_creating(lck);
2293 #endif // KMP_USE_DYNAMIC_LOCK 2297 void __kmpc_init_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2298 #if KMP_USE_DYNAMIC_LOCK 2300 KMP_DEBUG_ASSERT(__kmp_init_serial);
2301 if (__kmp_env_consistency_check && user_lock == NULL) {
2302 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
2304 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2306 #if OMPT_SUPPORT && OMPT_OPTIONAL 2308 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2310 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2311 if (ompt_enabled.ompt_callback_lock_init) {
2312 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2313 ompt_mutex_nest_lock, omp_lock_hint_none,
2314 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2319 #else // KMP_USE_DYNAMIC_LOCK 2321 static char const *
const func =
"omp_init_nest_lock";
2322 kmp_user_lock_p lck;
2323 KMP_DEBUG_ASSERT(__kmp_init_serial);
2325 if (__kmp_env_consistency_check) {
2326 if (user_lock == NULL) {
2327 KMP_FATAL(LockIsUninitialized, func);
2331 KMP_CHECK_USER_LOCK_INIT();
2333 if ((__kmp_user_lock_kind == lk_tas) &&
2334 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2335 OMP_NEST_LOCK_T_SIZE)) {
2336 lck = (kmp_user_lock_p)user_lock;
2339 else if ((__kmp_user_lock_kind == lk_futex) &&
2340 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2341 OMP_NEST_LOCK_T_SIZE)) {
2342 lck = (kmp_user_lock_p)user_lock;
2346 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2349 INIT_NESTED_LOCK(lck);
2350 __kmp_set_user_lock_location(lck, loc);
2352 #if OMPT_SUPPORT && OMPT_OPTIONAL 2354 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2356 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2357 if (ompt_enabled.ompt_callback_lock_init) {
2358 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2359 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2360 (ompt_wait_id_t)user_lock, codeptr);
2365 __kmp_itt_lock_creating(lck);
2368 #endif // KMP_USE_DYNAMIC_LOCK 2371 void __kmpc_destroy_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2372 #if KMP_USE_DYNAMIC_LOCK 2375 kmp_user_lock_p lck;
2376 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2377 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2379 lck = (kmp_user_lock_p)user_lock;
2381 __kmp_itt_lock_destroyed(lck);
2383 #if OMPT_SUPPORT && OMPT_OPTIONAL 2385 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2387 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2388 if (ompt_enabled.ompt_callback_lock_destroy) {
2389 kmp_user_lock_p lck;
2390 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2391 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2393 lck = (kmp_user_lock_p)user_lock;
2395 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2396 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2399 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2401 kmp_user_lock_p lck;
2403 if ((__kmp_user_lock_kind == lk_tas) &&
2404 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2405 lck = (kmp_user_lock_p)user_lock;
2408 else if ((__kmp_user_lock_kind == lk_futex) &&
2409 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2410 lck = (kmp_user_lock_p)user_lock;
2414 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_lock");
2417 #if OMPT_SUPPORT && OMPT_OPTIONAL 2419 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2421 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2422 if (ompt_enabled.ompt_callback_lock_destroy) {
2423 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2424 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2429 __kmp_itt_lock_destroyed(lck);
2433 if ((__kmp_user_lock_kind == lk_tas) &&
2434 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2438 else if ((__kmp_user_lock_kind == lk_futex) &&
2439 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2444 __kmp_user_lock_free(user_lock, gtid, lck);
2446 #endif // KMP_USE_DYNAMIC_LOCK 2450 void __kmpc_destroy_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2451 #if KMP_USE_DYNAMIC_LOCK 2454 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2455 __kmp_itt_lock_destroyed(ilk->lock);
2457 #if OMPT_SUPPORT && OMPT_OPTIONAL 2459 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2461 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2462 if (ompt_enabled.ompt_callback_lock_destroy) {
2463 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2464 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2467 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2469 #else // KMP_USE_DYNAMIC_LOCK 2471 kmp_user_lock_p lck;
2473 if ((__kmp_user_lock_kind == lk_tas) &&
2474 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2475 OMP_NEST_LOCK_T_SIZE)) {
2476 lck = (kmp_user_lock_p)user_lock;
2479 else if ((__kmp_user_lock_kind == lk_futex) &&
2480 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2481 OMP_NEST_LOCK_T_SIZE)) {
2482 lck = (kmp_user_lock_p)user_lock;
2486 lck = __kmp_lookup_user_lock(user_lock,
"omp_destroy_nest_lock");
2489 #if OMPT_SUPPORT && OMPT_OPTIONAL 2491 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2493 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2494 if (ompt_enabled.ompt_callback_lock_destroy) {
2495 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2496 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2501 __kmp_itt_lock_destroyed(lck);
2504 DESTROY_NESTED_LOCK(lck);
2506 if ((__kmp_user_lock_kind == lk_tas) &&
2507 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2508 OMP_NEST_LOCK_T_SIZE)) {
2512 else if ((__kmp_user_lock_kind == lk_futex) &&
2513 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2514 OMP_NEST_LOCK_T_SIZE)) {
2519 __kmp_user_lock_free(user_lock, gtid, lck);
2521 #endif // KMP_USE_DYNAMIC_LOCK 2524 void __kmpc_set_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2526 #if KMP_USE_DYNAMIC_LOCK 2527 int tag = KMP_EXTRACT_D_TAG(user_lock);
2529 __kmp_itt_lock_acquiring(
2533 #if OMPT_SUPPORT && OMPT_OPTIONAL 2535 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2537 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2538 if (ompt_enabled.ompt_callback_mutex_acquire) {
2539 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2540 ompt_mutex_lock, omp_lock_hint_none,
2541 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2545 #if KMP_USE_INLINED_TAS 2546 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2547 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2549 #elif KMP_USE_INLINED_FUTEX 2550 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2551 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2555 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2558 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2560 #if OMPT_SUPPORT && OMPT_OPTIONAL 2561 if (ompt_enabled.ompt_callback_mutex_acquired) {
2562 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2563 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2567 #else // KMP_USE_DYNAMIC_LOCK 2569 kmp_user_lock_p lck;
2571 if ((__kmp_user_lock_kind == lk_tas) &&
2572 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2573 lck = (kmp_user_lock_p)user_lock;
2576 else if ((__kmp_user_lock_kind == lk_futex) &&
2577 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2578 lck = (kmp_user_lock_p)user_lock;
2582 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_lock");
2586 __kmp_itt_lock_acquiring(lck);
2588 #if OMPT_SUPPORT && OMPT_OPTIONAL 2590 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2592 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2593 if (ompt_enabled.ompt_callback_mutex_acquire) {
2594 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2595 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2596 (ompt_wait_id_t)lck, codeptr);
2600 ACQUIRE_LOCK(lck, gtid);
2603 __kmp_itt_lock_acquired(lck);
2606 #if OMPT_SUPPORT && OMPT_OPTIONAL 2607 if (ompt_enabled.ompt_callback_mutex_acquired) {
2608 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2609 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2613 #endif // KMP_USE_DYNAMIC_LOCK 2616 void __kmpc_set_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2617 #if KMP_USE_DYNAMIC_LOCK 2620 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2622 #if OMPT_SUPPORT && OMPT_OPTIONAL 2624 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2626 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2627 if (ompt_enabled.enabled) {
2628 if (ompt_enabled.ompt_callback_mutex_acquire) {
2629 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2630 ompt_mutex_nest_lock, omp_lock_hint_none,
2631 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2636 int acquire_status =
2637 KMP_D_LOCK_FUNC(user_lock,
set)((kmp_dyna_lock_t *)user_lock, gtid);
2638 (void) acquire_status;
2640 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2643 #if OMPT_SUPPORT && OMPT_OPTIONAL 2644 if (ompt_enabled.enabled) {
2645 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2646 if (ompt_enabled.ompt_callback_mutex_acquired) {
2648 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2649 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2652 if (ompt_enabled.ompt_callback_nest_lock) {
2654 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2655 ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr);
2661 #else // KMP_USE_DYNAMIC_LOCK 2663 kmp_user_lock_p lck;
2665 if ((__kmp_user_lock_kind == lk_tas) &&
2666 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2667 OMP_NEST_LOCK_T_SIZE)) {
2668 lck = (kmp_user_lock_p)user_lock;
2671 else if ((__kmp_user_lock_kind == lk_futex) &&
2672 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2673 OMP_NEST_LOCK_T_SIZE)) {
2674 lck = (kmp_user_lock_p)user_lock;
2678 lck = __kmp_lookup_user_lock(user_lock,
"omp_set_nest_lock");
2682 __kmp_itt_lock_acquiring(lck);
2684 #if OMPT_SUPPORT && OMPT_OPTIONAL 2686 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2688 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2689 if (ompt_enabled.enabled) {
2690 if (ompt_enabled.ompt_callback_mutex_acquire) {
2691 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2692 ompt_mutex_nest_lock, omp_lock_hint_none,
2693 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr);
2698 ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status);
2701 __kmp_itt_lock_acquired(lck);
2704 #if OMPT_SUPPORT && OMPT_OPTIONAL 2705 if (ompt_enabled.enabled) {
2706 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2707 if (ompt_enabled.ompt_callback_mutex_acquired) {
2709 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2710 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2713 if (ompt_enabled.ompt_callback_nest_lock) {
2715 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2716 ompt_scope_begin, (ompt_wait_id_t)lck, codeptr);
2722 #endif // KMP_USE_DYNAMIC_LOCK 2725 void __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2726 #if KMP_USE_DYNAMIC_LOCK 2728 int tag = KMP_EXTRACT_D_TAG(user_lock);
2730 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2732 #if KMP_USE_INLINED_TAS 2733 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2734 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
2736 #elif KMP_USE_INLINED_FUTEX 2737 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2738 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
2742 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2745 #if OMPT_SUPPORT && OMPT_OPTIONAL 2747 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2749 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2750 if (ompt_enabled.ompt_callback_mutex_released) {
2751 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2752 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2756 #else // KMP_USE_DYNAMIC_LOCK 2758 kmp_user_lock_p lck;
2763 if ((__kmp_user_lock_kind == lk_tas) &&
2764 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2765 #if KMP_OS_LINUX && \ 2766 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2769 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2771 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2774 #if OMPT_SUPPORT && OMPT_OPTIONAL 2776 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2778 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2779 if (ompt_enabled.ompt_callback_mutex_released) {
2780 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2781 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2787 lck = (kmp_user_lock_p)user_lock;
2791 else if ((__kmp_user_lock_kind == lk_futex) &&
2792 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2793 lck = (kmp_user_lock_p)user_lock;
2797 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_lock");
2801 __kmp_itt_lock_releasing(lck);
2804 RELEASE_LOCK(lck, gtid);
2806 #if OMPT_SUPPORT && OMPT_OPTIONAL 2808 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2810 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2811 if (ompt_enabled.ompt_callback_mutex_released) {
2812 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2813 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
2817 #endif // KMP_USE_DYNAMIC_LOCK 2821 void __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2822 #if KMP_USE_DYNAMIC_LOCK 2825 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2827 int release_status =
2828 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2829 (void) release_status;
2831 #if OMPT_SUPPORT && OMPT_OPTIONAL 2833 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2835 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2836 if (ompt_enabled.enabled) {
2837 if (release_status == KMP_LOCK_RELEASED) {
2838 if (ompt_enabled.ompt_callback_mutex_released) {
2840 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2841 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
2843 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2845 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2846 ompt_scope_end, (ompt_wait_id_t)user_lock, codeptr);
2851 #else // KMP_USE_DYNAMIC_LOCK 2853 kmp_user_lock_p lck;
2857 if ((__kmp_user_lock_kind == lk_tas) &&
2858 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
2859 OMP_NEST_LOCK_T_SIZE)) {
2860 #if KMP_OS_LINUX && \ 2861 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2863 kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock;
2865 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2868 #if OMPT_SUPPORT && OMPT_OPTIONAL 2869 int release_status = KMP_LOCK_STILL_HELD;
2872 if (--(tl->lk.depth_locked) == 0) {
2873 TCW_4(tl->lk.poll, 0);
2874 #if OMPT_SUPPORT && OMPT_OPTIONAL 2875 release_status = KMP_LOCK_RELEASED;
2880 #if OMPT_SUPPORT && OMPT_OPTIONAL 2882 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2884 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2885 if (ompt_enabled.enabled) {
2886 if (release_status == KMP_LOCK_RELEASED) {
2887 if (ompt_enabled.ompt_callback_mutex_released) {
2889 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2890 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2892 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2894 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2895 ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr);
2902 lck = (kmp_user_lock_p)user_lock;
2906 else if ((__kmp_user_lock_kind == lk_futex) &&
2907 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
2908 OMP_NEST_LOCK_T_SIZE)) {
2909 lck = (kmp_user_lock_p)user_lock;
2913 lck = __kmp_lookup_user_lock(user_lock,
"omp_unset_nest_lock");
2917 __kmp_itt_lock_releasing(lck);
2921 release_status = RELEASE_NESTED_LOCK(lck, gtid);
2922 #if OMPT_SUPPORT && OMPT_OPTIONAL 2924 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2926 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2927 if (ompt_enabled.enabled) {
2928 if (release_status == KMP_LOCK_RELEASED) {
2929 if (ompt_enabled.ompt_callback_mutex_released) {
2931 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
2932 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
2934 }
else if (ompt_enabled.ompt_callback_nest_lock) {
2936 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2937 ompt_mutex_scope_end, (ompt_wait_id_t)lck, codeptr);
2942 #endif // KMP_USE_DYNAMIC_LOCK 2946 int __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
2949 #if KMP_USE_DYNAMIC_LOCK 2951 int tag = KMP_EXTRACT_D_TAG(user_lock);
2953 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2955 #if OMPT_SUPPORT && OMPT_OPTIONAL 2957 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2959 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2960 if (ompt_enabled.ompt_callback_mutex_acquire) {
2961 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2962 ompt_mutex_lock, omp_lock_hint_none,
2963 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
2967 #if KMP_USE_INLINED_TAS 2968 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2969 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
2971 #elif KMP_USE_INLINED_FUTEX 2972 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2973 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
2977 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2981 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2983 #if OMPT_SUPPORT && OMPT_OPTIONAL 2984 if (ompt_enabled.ompt_callback_mutex_acquired) {
2985 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2986 ompt_mutex_lock, (ompt_wait_id_t)user_lock, codeptr);
2992 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2997 #else // KMP_USE_DYNAMIC_LOCK 2999 kmp_user_lock_p lck;
3002 if ((__kmp_user_lock_kind == lk_tas) &&
3003 (
sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
3004 lck = (kmp_user_lock_p)user_lock;
3007 else if ((__kmp_user_lock_kind == lk_futex) &&
3008 (
sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
3009 lck = (kmp_user_lock_p)user_lock;
3013 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_lock");
3017 __kmp_itt_lock_acquiring(lck);
3019 #if OMPT_SUPPORT && OMPT_OPTIONAL 3021 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3023 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3024 if (ompt_enabled.ompt_callback_mutex_acquire) {
3025 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3026 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
3027 (ompt_wait_id_t)lck, codeptr);
3031 rc = TEST_LOCK(lck, gtid);
3034 __kmp_itt_lock_acquired(lck);
3036 __kmp_itt_lock_cancelled(lck);
3039 #if OMPT_SUPPORT && OMPT_OPTIONAL 3040 if (rc && ompt_enabled.ompt_callback_mutex_acquired) {
3041 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3042 ompt_mutex_lock, (ompt_wait_id_t)lck, codeptr);
3046 return (rc ? FTN_TRUE : FTN_FALSE);
3050 #endif // KMP_USE_DYNAMIC_LOCK 3054 int __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock) {
3055 #if KMP_USE_DYNAMIC_LOCK 3058 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
3060 #if OMPT_SUPPORT && OMPT_OPTIONAL 3062 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3064 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3065 if (ompt_enabled.ompt_callback_mutex_acquire) {
3066 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3067 ompt_mutex_nest_lock, omp_lock_hint_none,
3068 __ompt_get_mutex_impl_type(user_lock), (ompt_wait_id_t)user_lock,
3072 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
3075 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3077 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3080 #if OMPT_SUPPORT && OMPT_OPTIONAL 3081 if (ompt_enabled.enabled && rc) {
3083 if (ompt_enabled.ompt_callback_mutex_acquired) {
3085 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3086 ompt_mutex_nest_lock, (ompt_wait_id_t)user_lock, codeptr);
3089 if (ompt_enabled.ompt_callback_nest_lock) {
3091 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3092 ompt_scope_begin, (ompt_wait_id_t)user_lock, codeptr);
3099 #else // KMP_USE_DYNAMIC_LOCK 3101 kmp_user_lock_p lck;
3104 if ((__kmp_user_lock_kind == lk_tas) &&
3105 (
sizeof(lck->tas.lk.poll) +
sizeof(lck->tas.lk.depth_locked) <=
3106 OMP_NEST_LOCK_T_SIZE)) {
3107 lck = (kmp_user_lock_p)user_lock;
3110 else if ((__kmp_user_lock_kind == lk_futex) &&
3111 (
sizeof(lck->futex.lk.poll) +
sizeof(lck->futex.lk.depth_locked) <=
3112 OMP_NEST_LOCK_T_SIZE)) {
3113 lck = (kmp_user_lock_p)user_lock;
3117 lck = __kmp_lookup_user_lock(user_lock,
"omp_test_nest_lock");
3121 __kmp_itt_lock_acquiring(lck);
3124 #if OMPT_SUPPORT && OMPT_OPTIONAL 3126 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3128 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3129 if (ompt_enabled.enabled) &&
3130 ompt_enabled.ompt_callback_mutex_acquire) {
3131 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3132 ompt_mutex_nest_lock, omp_lock_hint_none,
3133 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)lck, codeptr);
3137 rc = TEST_NESTED_LOCK(lck, gtid);
3140 __kmp_itt_lock_acquired(lck);
3142 __kmp_itt_lock_cancelled(lck);
3145 #if OMPT_SUPPORT && OMPT_OPTIONAL 3146 if (ompt_enabled.enabled && rc) {
3148 if (ompt_enabled.ompt_callback_mutex_acquired) {
3150 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3151 ompt_mutex_nest_lock, (ompt_wait_id_t)lck, codeptr);
3154 if (ompt_enabled.ompt_callback_nest_lock) {
3156 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3157 ompt_mutex_scope_begin, (ompt_wait_id_t)lck, codeptr);
3166 #endif // KMP_USE_DYNAMIC_LOCK 3176 #define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \ 3177 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod)) 3179 #define __KMP_GET_REDUCTION_METHOD(gtid) \ 3180 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) 3186 static __forceinline
void 3187 __kmp_enter_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3188 kmp_critical_name *crit) {
3194 kmp_user_lock_p lck;
3196 #if KMP_USE_DYNAMIC_LOCK 3198 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
3201 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3202 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0,
3203 KMP_GET_D_TAG(__kmp_user_lock_seq));
3205 __kmp_init_indirect_csptr(crit, loc, global_tid,
3206 KMP_GET_I_TAG(__kmp_user_lock_seq));
3212 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3213 lck = (kmp_user_lock_p)lk;
3214 KMP_DEBUG_ASSERT(lck != NULL);
3215 if (__kmp_env_consistency_check) {
3216 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3218 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
3220 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3222 KMP_DEBUG_ASSERT(lck != NULL);
3223 if (__kmp_env_consistency_check) {
3224 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3226 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
3229 #else // KMP_USE_DYNAMIC_LOCK 3234 if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) {
3235 lck = (kmp_user_lock_p)crit;
3237 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
3239 KMP_DEBUG_ASSERT(lck != NULL);
3241 if (__kmp_env_consistency_check)
3242 __kmp_push_sync(global_tid, ct_critical, loc, lck);
3244 __kmp_acquire_user_lock_with_checks(lck, global_tid);
3246 #endif // KMP_USE_DYNAMIC_LOCK 3250 static __forceinline
void 3251 __kmp_end_critical_section_reduce_block(
ident_t *loc, kmp_int32 global_tid,
3252 kmp_critical_name *crit) {
3254 kmp_user_lock_p lck;
3256 #if KMP_USE_DYNAMIC_LOCK 3258 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3259 lck = (kmp_user_lock_p)crit;
3260 if (__kmp_env_consistency_check)
3261 __kmp_pop_sync(global_tid, ct_critical, loc);
3262 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
3264 kmp_indirect_lock_t *ilk =
3265 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
3266 if (__kmp_env_consistency_check)
3267 __kmp_pop_sync(global_tid, ct_critical, loc);
3268 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3271 #else // KMP_USE_DYNAMIC_LOCK 3276 if (__kmp_base_user_lock_size > 32) {
3277 lck = *((kmp_user_lock_p *)crit);
3278 KMP_ASSERT(lck != NULL);
3280 lck = (kmp_user_lock_p)crit;
3283 if (__kmp_env_consistency_check)
3284 __kmp_pop_sync(global_tid, ct_critical, loc);
3286 __kmp_release_user_lock_with_checks(lck, global_tid);
3288 #endif // KMP_USE_DYNAMIC_LOCK 3292 static __forceinline
int 3293 __kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p,
3298 if (th->th.th_teams_microtask) {
3299 *team_p = team = th->th.th_team;
3300 if (team->t.t_level == th->th.th_teams_level) {
3302 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
3304 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
3305 th->th.th_team = team->t.t_parent;
3306 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3307 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3308 *task_state = th->th.th_task_state;
3309 th->th.th_task_state = 0;
3317 static __forceinline
void 3318 __kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team,
int task_state) {
3320 th->th.th_info.ds.ds_tid = 0;
3321 th->th.th_team = team;
3322 th->th.th_team_nproc = team->t.t_nproc;
3323 th->th.th_task_team = team->t.t_task_team[task_state];
3324 th->th.th_task_state = task_state;
3346 size_t reduce_size,
void *reduce_data,
3347 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3348 kmp_critical_name *lck) {
3352 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3356 int teams_swapped = 0, task_state;
3358 KA_TRACE(10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3366 if (!TCR_4(__kmp_init_parallel))
3367 __kmp_parallel_initialize();
3370 #if KMP_USE_DYNAMIC_LOCK 3371 if (__kmp_env_consistency_check)
3372 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3374 if (__kmp_env_consistency_check)
3375 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3379 th = __kmp_thread_from_gtid(global_tid);
3380 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3381 #endif // OMP_40_ENABLED 3399 packed_reduction_method = __kmp_determine_reduction_method(
3400 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3401 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3403 if (packed_reduction_method == critical_reduce_block) {
3405 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3408 }
else if (packed_reduction_method == empty_reduce_block) {
3414 }
else if (packed_reduction_method == atomic_reduce_block) {
3424 if (__kmp_env_consistency_check)
3425 __kmp_pop_sync(global_tid, ct_reduce, loc);
3427 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3428 tree_reduce_block)) {
3448 ompt_frame_t *ompt_frame;
3449 if (ompt_enabled.enabled) {
3450 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3451 if (ompt_frame->enter_frame.ptr == NULL)
3452 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3453 OMPT_STORE_RETURN_ADDRESS(global_tid);
3457 __kmp_threads[global_tid]->th.th_ident = loc;
3460 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3461 global_tid, FALSE, reduce_size, reduce_data, reduce_func);
3462 retval = (retval != 0) ? (0) : (1);
3463 #if OMPT_SUPPORT && OMPT_OPTIONAL 3464 if (ompt_enabled.enabled) {
3465 ompt_frame->enter_frame = ompt_data_none;
3471 if (__kmp_env_consistency_check) {
3473 __kmp_pop_sync(global_tid, ct_reduce, loc);
3483 if (teams_swapped) {
3484 __kmp_restore_swapped_teams(th, team, task_state);
3489 (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3490 global_tid, packed_reduction_method, retval));
3504 kmp_critical_name *lck) {
3506 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3508 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3510 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3512 if (packed_reduction_method == critical_reduce_block) {
3514 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3516 }
else if (packed_reduction_method == empty_reduce_block) {
3521 }
else if (packed_reduction_method == atomic_reduce_block) {
3528 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3529 tree_reduce_block)) {
3539 if (__kmp_env_consistency_check)
3540 __kmp_pop_sync(global_tid, ct_reduce, loc);
3542 KA_TRACE(10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3543 global_tid, packed_reduction_method));
3566 size_t reduce_size,
void *reduce_data,
3567 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
3568 kmp_critical_name *lck) {
3571 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3575 int teams_swapped = 0, task_state;
3578 KA_TRACE(10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid));
3586 if (!TCR_4(__kmp_init_parallel))
3587 __kmp_parallel_initialize();
3590 #if KMP_USE_DYNAMIC_LOCK 3591 if (__kmp_env_consistency_check)
3592 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3594 if (__kmp_env_consistency_check)
3595 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3599 th = __kmp_thread_from_gtid(global_tid);
3600 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3601 #endif // OMP_40_ENABLED 3603 packed_reduction_method = __kmp_determine_reduction_method(
3604 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3605 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3607 if (packed_reduction_method == critical_reduce_block) {
3609 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3612 }
else if (packed_reduction_method == empty_reduce_block) {
3618 }
else if (packed_reduction_method == atomic_reduce_block) {
3622 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3623 tree_reduce_block)) {
3629 ompt_frame_t *ompt_frame;
3630 if (ompt_enabled.enabled) {
3631 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3632 if (ompt_frame->enter_frame.ptr == NULL)
3633 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3634 OMPT_STORE_RETURN_ADDRESS(global_tid);
3638 __kmp_threads[global_tid]->th.th_ident =
3642 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3643 global_tid, TRUE, reduce_size, reduce_data, reduce_func);
3644 retval = (retval != 0) ? (0) : (1);
3645 #if OMPT_SUPPORT && OMPT_OPTIONAL 3646 if (ompt_enabled.enabled) {
3647 ompt_frame->enter_frame = ompt_data_none;
3653 if (__kmp_env_consistency_check) {
3655 __kmp_pop_sync(global_tid, ct_reduce, loc);
3665 if (teams_swapped) {
3666 __kmp_restore_swapped_teams(th, team, task_state);
3671 (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
3672 global_tid, packed_reduction_method, retval));
3688 kmp_critical_name *lck) {
3690 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3694 int teams_swapped = 0, task_state;
3697 KA_TRACE(10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid));
3700 th = __kmp_thread_from_gtid(global_tid);
3701 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3702 #endif // OMP_40_ENABLED 3704 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3709 if (packed_reduction_method == critical_reduce_block) {
3711 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3715 ompt_frame_t *ompt_frame;
3716 if (ompt_enabled.enabled) {
3717 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3718 if (ompt_frame->enter_frame.ptr == NULL)
3719 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3720 OMPT_STORE_RETURN_ADDRESS(global_tid);
3724 __kmp_threads[global_tid]->th.th_ident = loc;
3726 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3727 #if OMPT_SUPPORT && OMPT_OPTIONAL 3728 if (ompt_enabled.enabled) {
3729 ompt_frame->enter_frame = ompt_data_none;
3733 }
else if (packed_reduction_method == empty_reduce_block) {
3739 ompt_frame_t *ompt_frame;
3740 if (ompt_enabled.enabled) {
3741 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3742 if (ompt_frame->enter_frame.ptr == NULL)
3743 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3744 OMPT_STORE_RETURN_ADDRESS(global_tid);
3748 __kmp_threads[global_tid]->th.th_ident = loc;
3750 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3751 #if OMPT_SUPPORT && OMPT_OPTIONAL 3752 if (ompt_enabled.enabled) {
3753 ompt_frame->enter_frame = ompt_data_none;
3757 }
else if (packed_reduction_method == atomic_reduce_block) {
3760 ompt_frame_t *ompt_frame;
3761 if (ompt_enabled.enabled) {
3762 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3763 if (ompt_frame->enter_frame.ptr == NULL)
3764 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3765 OMPT_STORE_RETURN_ADDRESS(global_tid);
3770 __kmp_threads[global_tid]->th.th_ident = loc;
3772 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
3773 #if OMPT_SUPPORT && OMPT_OPTIONAL 3774 if (ompt_enabled.enabled) {
3775 ompt_frame->enter_frame = ompt_data_none;
3779 }
else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3780 tree_reduce_block)) {
3783 __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3792 if (teams_swapped) {
3793 __kmp_restore_swapped_teams(th, team, task_state);
3797 if (__kmp_env_consistency_check)
3798 __kmp_pop_sync(global_tid, ct_reduce, loc);
3800 KA_TRACE(10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n",
3801 global_tid, packed_reduction_method));
3806 #undef __KMP_GET_REDUCTION_METHOD 3807 #undef __KMP_SET_REDUCTION_METHOD 3811 kmp_uint64 __kmpc_get_taskid() {
3816 gtid = __kmp_get_gtid();
3820 thread = __kmp_thread_from_gtid(gtid);
3821 return thread->th.th_current_task->td_task_id;
3825 kmp_uint64 __kmpc_get_parent_taskid() {
3829 kmp_taskdata_t *parent_task;
3831 gtid = __kmp_get_gtid();
3835 thread = __kmp_thread_from_gtid(gtid);
3836 parent_task = thread->th.th_current_task->td_parent;
3837 return (parent_task == NULL ? 0 : parent_task->td_task_id);
3853 void __kmpc_doacross_init(
ident_t *loc,
int gtid,
int num_dims,
3854 const struct kmp_dim *dims) {
3856 kmp_int64 last, trace_count;
3857 kmp_info_t *th = __kmp_threads[gtid];
3858 kmp_team_t *team = th->th.th_team;
3860 kmp_disp_t *pr_buf = th->th.th_dispatch;
3861 dispatch_shared_info_t *sh_buf;
3865 (
"__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
3866 gtid, num_dims, !team->t.t_serialized));
3867 KMP_DEBUG_ASSERT(dims != NULL);
3868 KMP_DEBUG_ASSERT(num_dims > 0);
3870 if (team->t.t_serialized) {
3871 KA_TRACE(20, (
"__kmpc_doacross_init() exit: serialized team\n"));
3874 KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
3875 idx = pr_buf->th_doacross_buf_idx++;
3877 sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
3880 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
3881 pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc(
3882 th,
sizeof(kmp_int64) * (4 * num_dims + 1));
3883 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3884 pr_buf->th_doacross_info[0] =
3885 (kmp_int64)num_dims;
3888 pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
3889 pr_buf->th_doacross_info[2] = dims[0].lo;
3890 pr_buf->th_doacross_info[3] = dims[0].up;
3891 pr_buf->th_doacross_info[4] = dims[0].st;
3893 for (j = 1; j < num_dims; ++j) {
3896 if (dims[j].st == 1) {
3898 range_length = dims[j].up - dims[j].lo + 1;
3900 if (dims[j].st > 0) {
3901 KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
3902 range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
3904 KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
3906 (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
3909 pr_buf->th_doacross_info[last++] = range_length;
3910 pr_buf->th_doacross_info[last++] = dims[j].lo;
3911 pr_buf->th_doacross_info[last++] = dims[j].up;
3912 pr_buf->th_doacross_info[last++] = dims[j].st;
3917 if (dims[0].st == 1) {
3918 trace_count = dims[0].up - dims[0].lo + 1;
3919 }
else if (dims[0].st > 0) {
3920 KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
3921 trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
3923 KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
3924 trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
3926 for (j = 1; j < num_dims; ++j) {
3927 trace_count *= pr_buf->th_doacross_info[4 * j + 1];
3929 KMP_DEBUG_ASSERT(trace_count > 0);
3933 if (idx != sh_buf->doacross_buf_idx) {
3935 __kmp_wait_yield_4((
volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
3942 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
3943 (
volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
3945 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
3946 (
volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
3948 if (flags == NULL) {
3950 size_t size = trace_count / 8 + 8;
3951 flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
3953 sh_buf->doacross_flags = flags;
3954 }
else if (flags == (kmp_uint32 *)1) {
3957 while (*(
volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
3959 while (*(
volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
3966 KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1);
3967 pr_buf->th_doacross_flags =
3968 sh_buf->doacross_flags;
3970 KA_TRACE(20, (
"__kmpc_doacross_init() exit: T#%d\n", gtid));
3973 void __kmpc_doacross_wait(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
3974 kmp_int32 shft, num_dims, i;
3976 kmp_int64 iter_number;
3977 kmp_info_t *th = __kmp_threads[gtid];
3978 kmp_team_t *team = th->th.th_team;
3980 kmp_int64 lo, up, st;
3982 KA_TRACE(20, (
"__kmpc_doacross_wait() enter: called T#%d\n", gtid));
3983 if (team->t.t_serialized) {
3984 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: serialized team\n"));
3989 pr_buf = th->th.th_dispatch;
3990 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
3991 num_dims = pr_buf->th_doacross_info[0];
3992 lo = pr_buf->th_doacross_info[2];
3993 up = pr_buf->th_doacross_info[3];
3994 st = pr_buf->th_doacross_info[4];
3996 if (vec[0] < lo || vec[0] > up) {
3997 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 3998 "bounds [%lld,%lld]\n",
3999 gtid, vec[0], lo, up));
4002 iter_number = vec[0] - lo;
4003 }
else if (st > 0) {
4004 if (vec[0] < lo || vec[0] > up) {
4005 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4006 "bounds [%lld,%lld]\n",
4007 gtid, vec[0], lo, up));
4010 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4012 if (vec[0] > lo || vec[0] < up) {
4013 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4014 "bounds [%lld,%lld]\n",
4015 gtid, vec[0], lo, up));
4018 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4020 for (i = 1; i < num_dims; ++i) {
4022 kmp_int32 j = i * 4;
4023 ln = pr_buf->th_doacross_info[j + 1];
4024 lo = pr_buf->th_doacross_info[j + 2];
4025 up = pr_buf->th_doacross_info[j + 3];
4026 st = pr_buf->th_doacross_info[j + 4];
4028 if (vec[i] < lo || vec[i] > up) {
4029 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4030 "bounds [%lld,%lld]\n",
4031 gtid, vec[i], lo, up));
4035 }
else if (st > 0) {
4036 if (vec[i] < lo || vec[i] > up) {
4037 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4038 "bounds [%lld,%lld]\n",
4039 gtid, vec[i], lo, up));
4042 iter = (kmp_uint64)(vec[i] - lo) / st;
4044 if (vec[i] > lo || vec[i] < up) {
4045 KA_TRACE(20, (
"__kmpc_doacross_wait() exit: T#%d iter %lld is out of " 4046 "bounds [%lld,%lld]\n",
4047 gtid, vec[i], lo, up));
4050 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4052 iter_number = iter + ln * iter_number;
4054 shft = iter_number % 32;
4057 while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
4062 (
"__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
4063 gtid, (iter_number << 5) + shft));
4066 void __kmpc_doacross_post(
ident_t *loc,
int gtid,
const kmp_int64 *vec) {
4067 kmp_int32 shft, num_dims, i;
4069 kmp_int64 iter_number;
4070 kmp_info_t *th = __kmp_threads[gtid];
4071 kmp_team_t *team = th->th.th_team;
4075 KA_TRACE(20, (
"__kmpc_doacross_post() enter: called T#%d\n", gtid));
4076 if (team->t.t_serialized) {
4077 KA_TRACE(20, (
"__kmpc_doacross_post() exit: serialized team\n"));
4083 pr_buf = th->th.th_dispatch;
4084 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4085 num_dims = pr_buf->th_doacross_info[0];
4086 lo = pr_buf->th_doacross_info[2];
4087 st = pr_buf->th_doacross_info[4];
4089 iter_number = vec[0] - lo;
4090 }
else if (st > 0) {
4091 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4093 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4095 for (i = 1; i < num_dims; ++i) {
4097 kmp_int32 j = i * 4;
4098 ln = pr_buf->th_doacross_info[j + 1];
4099 lo = pr_buf->th_doacross_info[j + 2];
4100 st = pr_buf->th_doacross_info[j + 4];
4103 }
else if (st > 0) {
4104 iter = (kmp_uint64)(vec[i] - lo) / st;
4106 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4108 iter_number = iter + ln * iter_number;
4110 shft = iter_number % 32;
4114 if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
4115 KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
4116 KA_TRACE(20, (
"__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4117 (iter_number << 5) + shft));
4120 void __kmpc_doacross_fini(
ident_t *loc,
int gtid) {
4122 kmp_info_t *th = __kmp_threads[gtid];
4123 kmp_team_t *team = th->th.th_team;
4124 kmp_disp_t *pr_buf = th->th.th_dispatch;
4126 KA_TRACE(20, (
"__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4127 if (team->t.t_serialized) {
4128 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: serialized team %p\n", team));
4131 num_done = KMP_TEST_THEN_INC32((kmp_int32 *)pr_buf->th_doacross_info[1]) + 1;
4132 if (num_done == th->th.th_team_nproc) {
4134 int idx = pr_buf->th_doacross_buf_idx - 1;
4135 dispatch_shared_info_t *sh_buf =
4136 &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4137 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] ==
4138 (kmp_int64)&sh_buf->doacross_num_done);
4139 KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done);
4140 KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
4141 __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags));
4142 sh_buf->doacross_flags = NULL;
4143 sh_buf->doacross_num_done = 0;
4144 sh_buf->doacross_buf_idx +=
4145 __kmp_dispatch_num_buffers;
4148 pr_buf->th_doacross_flags = NULL;
4149 __kmp_thread_free(th, (
void *)pr_buf->th_doacross_info);
4150 pr_buf->th_doacross_info = NULL;
4151 KA_TRACE(20, (
"__kmpc_doacross_fini() exit: T#%d\n", gtid));
4156 int __kmpc_get_target_offload(
void) {
4157 if (!__kmp_init_serial) {
4158 __kmp_serial_initialize();
4160 return __kmp_target_offload;
4162 #endif // OMP_50_ENABLED kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
stats_state_e
the states which a thread can be in
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)