20 #include "kmp_error.h" 21 #include "kmp_stats.h" 24 #include "ompt-internal.h" 25 #include "ompt-specific.h" 28 #define MAX_MESSAGE 512 49 if (__kmp_ignore_mppbeg() == FALSE) {
50 __kmp_internal_begin();
52 KC_TRACE( 10, (
"__kmpc_begin: called\n" ) );
70 if (__kmp_ignore_mppend() == FALSE) {
71 KC_TRACE( 10, (
"__kmpc_end: called\n" ) );
72 KA_TRACE( 30, (
"__kmpc_end\n" ));
74 __kmp_internal_end_thread( -1 );
100 kmp_int32 gtid = __kmp_entry_gtid();
102 KC_TRACE( 10, (
"__kmpc_global_thread_num: T#%d\n", gtid ) );
123 KC_TRACE( 10, (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
125 return TCR_4(__kmp_nth);
137 KC_TRACE( 10, (
"__kmpc_bound_thread_num: called\n" ) );
138 return __kmp_tid_from_gtid( __kmp_entry_gtid() );
149 KC_TRACE( 10, (
"__kmpc_bound_num_threads: called\n" ) );
151 return __kmp_entry_thread() -> th.th_team -> t.t_nproc;
173 if (__kmp_par_range == 0) {
180 semi2 = strchr(semi2,
';');
184 semi2 = strchr(semi2 + 1,
';');
188 if (__kmp_par_range_filename[0]) {
189 const char *name = semi2 - 1;
190 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
193 if ((*name ==
'/') || (*name ==
';')) {
196 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
197 return __kmp_par_range < 0;
200 semi3 = strchr(semi2 + 1,
';');
201 if (__kmp_par_range_routine[0]) {
202 if ((semi3 != NULL) && (semi3 > semi2)
203 && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
204 return __kmp_par_range < 0;
207 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
208 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
209 return __kmp_par_range > 0;
211 return __kmp_par_range < 0;
227 return __kmp_entry_thread() -> th.th_root -> r.r_active;
242 KA_TRACE( 20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
243 global_tid, num_threads ) );
245 __kmp_push_num_threads( loc, global_tid, num_threads );
249 __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid )
251 KA_TRACE( 20, (
"__kmpc_pop_num_threads: enter\n" ) );
260 __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind )
262 KA_TRACE( 20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n",
263 global_tid, proc_bind ) );
265 __kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind );
283 int gtid = __kmp_entry_gtid();
285 #if (KMP_STATS_ENABLED) 301 va_start( ap, microtask );
304 int tid = __kmp_tid_from_gtid( gtid );
305 kmp_info_t *master_th = __kmp_threads[ gtid ];
306 kmp_team_t *parent_team = master_th->th.th_team;
308 parent_team->t.t_implicit_task_taskdata[tid].
309 ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
313 #if INCLUDE_SSC_MARKS 316 __kmp_fork_call( loc, gtid, fork_context_intel,
319 VOLATILE_CAST(
void *) microtask,
321 VOLATILE_CAST(microtask_t) microtask,
322 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
324 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
330 #if INCLUDE_SSC_MARKS 333 __kmp_join_call( loc, gtid
343 parent_team->t.t_implicit_task_taskdata[tid].
344 ompt_task_info.frame.reenter_runtime_frame = 0;
348 #if (KMP_STATS_ENABLED) 369 KA_TRACE( 20, (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
370 global_tid, num_teams, num_threads ) );
372 __kmp_push_num_teams( loc, global_tid, num_teams, num_threads );
387 int gtid = __kmp_entry_gtid();
388 kmp_info_t *this_thr = __kmp_threads[ gtid ];
390 va_start( ap, microtask );
395 this_thr->th.th_teams_microtask = microtask;
396 this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level;
399 kmp_team_t *parent_team = this_thr->th.th_team;
400 int tid = __kmp_tid_from_gtid( gtid );
402 parent_team->t.t_implicit_task_taskdata[tid].
403 ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
408 if ( this_thr->th.th_teams_size.nteams == 0 ) {
409 __kmp_push_num_teams( loc, gtid, 0, 0 );
411 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
412 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
413 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
415 __kmp_fork_call( loc, gtid, fork_context_intel,
418 VOLATILE_CAST(
void *) microtask,
420 VOLATILE_CAST(microtask_t) __kmp_teams_master,
421 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
422 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
428 __kmp_join_call( loc, gtid
436 parent_team->t.t_implicit_task_taskdata[tid].
437 ompt_task_info.frame.reenter_runtime_frame = NULL;
441 this_thr->th.th_teams_microtask = NULL;
442 this_thr->th.th_teams_level = 0;
443 *(kmp_int64*)(&this_thr->th.th_teams_size) = 0L;
456 __kmpc_invoke_task_func(
int gtid )
458 return __kmp_invoke_task_func( gtid );
476 __kmp_serialized_parallel(loc, global_tid);
491 kmp_internal_control_t *top;
492 kmp_info_t *this_thr;
493 kmp_team_t *serial_team;
495 KC_TRACE( 10, (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) );
503 if( ! TCR_4( __kmp_init_parallel ) )
504 __kmp_parallel_initialize();
506 this_thr = __kmp_threads[ global_tid ];
507 serial_team = this_thr->th.th_serial_team;
510 kmp_task_team_t * task_team = this_thr->th.th_task_team;
513 if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks )
514 __kmp_task_team_wait(this_thr, serial_team, NULL );
518 KMP_DEBUG_ASSERT( serial_team );
519 KMP_ASSERT( serial_team -> t.t_serialized );
520 KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
521 KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
522 KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
523 KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
526 top = serial_team -> t.t_control_stack_top;
527 if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) {
528 copy_icvs( &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs, top );
529 serial_team -> t.t_control_stack_top = top -> next;
534 serial_team -> t.t_level--;
537 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
539 dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer;
540 serial_team->t.t_dispatch->th_disp_buffer =
541 serial_team->t.t_dispatch->th_disp_buffer->next;
542 __kmp_free( disp_buffer );
545 -- serial_team -> t.t_serialized;
546 if ( serial_team -> t.t_serialized == 0 ) {
550 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 551 if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {
552 __kmp_clear_x87_fpu_status_word();
553 __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
554 __kmp_load_mxcsr( &serial_team->t.t_mxcsr );
558 this_thr -> th.th_team = serial_team -> t.t_parent;
559 this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;
562 this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc;
563 this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0];
564 this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized;
567 this_thr -> th.th_dispatch = & this_thr -> th.th_team ->
568 t.t_dispatch[ serial_team -> t.t_master_tid ];
570 __kmp_pop_current_task_from_thread( this_thr );
572 KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 );
573 this_thr -> th.th_current_task -> td_flags.executing = 1;
575 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
577 this_thr->th.th_task_team = this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
578 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n",
579 global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
582 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
583 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
584 global_tid, serial_team, serial_team -> t.t_serialized ) );
589 kmp_uint64 cur_time = 0;
591 if ( __itt_get_timestamp_ptr ) {
592 cur_time = __itt_get_timestamp();
595 if ( this_thr->th.th_team->t.t_level == 0
597 && this_thr->th.th_teams_microtask == NULL
601 this_thr->th.th_ident = loc;
602 if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) &&
603 ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
605 __kmp_itt_frame_submit( global_tid, this_thr->th.th_frame_time_serialized,
606 cur_time, 0, loc, this_thr->th.th_team_nproc, 0 );
607 if ( __kmp_forkjoin_frames_mode == 3 )
609 __kmp_itt_frame_submit( global_tid, serial_team->t.t_region_time,
610 cur_time, 0, loc, this_thr->th.th_team_nproc, 2 );
611 }
else if ( ( __itt_frame_end_v3_ptr || KMP_ITT_DEBUG ) &&
612 ! __kmp_forkjoin_frames_mode && __kmp_forkjoin_frames )
614 __kmp_itt_region_joined( global_tid, 1 );
618 if ( __kmp_env_consistency_check )
619 __kmp_pop_parallel( global_tid, NULL );
633 KC_TRACE( 10, (
"__kmpc_flush: called\n" ) );
638 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 ) 651 if ( ! __kmp_cpuinfo.initialized ) {
652 __kmp_query_cpuid( & __kmp_cpuinfo );
654 if ( ! __kmp_cpuinfo.sse2 ) {
657 #if KMP_COMPILER_ICC || KMP_COMPILER_MSVC 660 __sync_synchronize();
661 #endif // KMP_COMPILER_ICC 664 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64) 680 #error Unknown or unsupported architecture 701 KC_TRACE( 10, (
"__kmpc_barrier: called T#%d\n", global_tid ) );
703 if (! TCR_4(__kmp_init_parallel))
704 __kmp_parallel_initialize();
706 if ( __kmp_env_consistency_check ) {
708 KMP_WARNING( ConstructIdentInvalid );
711 __kmp_check_barrier( global_tid, ct_barrier, loc );
714 __kmp_threads[ global_tid ]->th.th_ident = loc;
722 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
738 KC_TRACE( 10, (
"__kmpc_master: called T#%d\n", global_tid ) );
740 if( ! TCR_4( __kmp_init_parallel ) )
741 __kmp_parallel_initialize();
743 if( KMP_MASTER_GTID( global_tid )) {
748 #if OMPT_SUPPORT && OMPT_TRACE 751 ompt_callbacks.ompt_callback(ompt_event_master_begin)) {
752 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
753 kmp_team_t *team = this_thr -> th.th_team;
755 int tid = __kmp_tid_from_gtid( global_tid );
756 ompt_callbacks.ompt_callback(ompt_event_master_begin)(
757 team->t.ompt_team_info.parallel_id,
758 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
763 if ( __kmp_env_consistency_check ) {
764 #if KMP_USE_DYNAMIC_LOCK 766 __kmp_push_sync( global_tid, ct_master, loc, NULL, 0 );
768 __kmp_check_sync( global_tid, ct_master, loc, NULL, 0 );
771 __kmp_push_sync( global_tid, ct_master, loc, NULL );
773 __kmp_check_sync( global_tid, ct_master, loc, NULL );
791 KC_TRACE( 10, (
"__kmpc_end_master: called T#%d\n", global_tid ) );
793 KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
796 #if OMPT_SUPPORT && OMPT_TRACE 797 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
798 kmp_team_t *team = this_thr -> th.th_team;
800 ompt_callbacks.ompt_callback(ompt_event_master_end)) {
801 int tid = __kmp_tid_from_gtid( global_tid );
802 ompt_callbacks.ompt_callback(ompt_event_master_end)(
803 team->t.ompt_team_info.parallel_id,
804 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
808 if ( __kmp_env_consistency_check ) {
810 KMP_WARNING( ThreadIdentInvalid );
812 if( KMP_MASTER_GTID( global_tid ))
813 __kmp_pop_sync( global_tid, ct_master, loc );
829 KMP_DEBUG_ASSERT( __kmp_init_serial );
831 KC_TRACE( 10, (
"__kmpc_ordered: called T#%d\n", gtid ));
833 if (! TCR_4(__kmp_init_parallel))
834 __kmp_parallel_initialize();
837 __kmp_itt_ordered_prep( gtid );
841 th = __kmp_threads[ gtid ];
843 #if OMPT_SUPPORT && OMPT_TRACE 846 th->th.ompt_thread_info.wait_id = (uint64_t) loc;
847 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
850 if (ompt_callbacks.ompt_callback(ompt_event_wait_ordered)) {
851 ompt_callbacks.ompt_callback(ompt_event_wait_ordered)(
852 th->th.ompt_thread_info.wait_id);
857 if ( th -> th.th_dispatch -> th_deo_fcn != 0 )
858 (*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc );
860 __kmp_parallel_deo( & gtid, & cid, loc );
862 #if OMPT_SUPPORT && OMPT_TRACE 865 th->th.ompt_thread_info.state = ompt_state_work_parallel;
866 th->th.ompt_thread_info.wait_id = 0;
869 if (ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)) {
870 ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)(
871 th->th.ompt_thread_info.wait_id);
877 __kmp_itt_ordered_start( gtid );
894 KC_TRACE( 10, (
"__kmpc_end_ordered: called T#%d\n", gtid ) );
897 __kmp_itt_ordered_end( gtid );
901 th = __kmp_threads[ gtid ];
903 if ( th -> th.th_dispatch -> th_dxo_fcn != 0 )
904 (*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc );
906 __kmp_parallel_dxo( & gtid, & cid, loc );
908 #if OMPT_SUPPORT && OMPT_BLAME 910 ompt_callbacks.ompt_callback(ompt_event_release_ordered)) {
911 ompt_callbacks.ompt_callback(ompt_event_release_ordered)(
912 th->th.ompt_thread_info.wait_id);
917 #if KMP_USE_DYNAMIC_LOCK 919 static __forceinline
void 920 __kmp_init_indirect_csptr(kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid, kmp_indirect_locktag_t tag)
924 kmp_indirect_lock_t **lck;
925 lck = (kmp_indirect_lock_t **)crit;
926 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
927 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
928 KMP_SET_I_LOCK_LOCATION(ilk, loc);
929 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
930 KA_TRACE(20, (
"__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
932 __kmp_itt_critical_creating(ilk->lock, loc);
934 int status = KMP_COMPARE_AND_STORE_PTR(lck, 0, ilk);
937 __kmp_itt_critical_destroyed(ilk->lock);
942 KMP_DEBUG_ASSERT(*lck != NULL);
946 #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) { \ 947 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 948 if (l->lk.poll != KMP_LOCK_FREE(tas) || \ 949 ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \ 951 KMP_FSYNC_PREPARE(l); \ 952 KMP_INIT_YIELD(spins); \ 953 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 956 KMP_YIELD_SPIN(spins); \ 958 while (l->lk.poll != KMP_LOCK_FREE(tas) || \ 959 ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \ 960 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 963 KMP_YIELD_SPIN(spins); \ 967 KMP_FSYNC_ACQUIRED(l); \ 971 #define KMP_TEST_TAS_LOCK(lock, gtid, rc) { \ 972 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ 973 rc = l->lk.poll == KMP_LOCK_FREE(tas) && \ 974 KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas)); \ 978 #define KMP_RELEASE_TAS_LOCK(lock, gtid) { \ 979 TCW_4(((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); \ 986 # include <sys/syscall.h> 988 # define FUTEX_WAIT 0 991 # define FUTEX_WAKE 1 995 #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) { \ 996 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 997 kmp_int32 gtid_code = (gtid+1) << 1; \ 999 KMP_FSYNC_PREPARE(ftx); \ 1000 kmp_int32 poll_val; \ 1001 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ 1002 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ 1003 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ 1005 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, poll_val | KMP_LOCK_BUSY(1, futex))) { \ 1008 poll_val |= KMP_LOCK_BUSY(1, futex); \ 1011 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, NULL, NULL, 0)) != 0) { \ 1016 KMP_FSYNC_ACQUIRED(ftx); \ 1020 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) { \ 1021 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1022 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1, futex) << 1)) { \ 1023 KMP_FSYNC_ACQUIRED(ftx); \ 1031 #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) { \ 1032 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ 1034 KMP_FSYNC_RELEASING(ftx); \ 1035 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ 1036 if (KMP_LOCK_STRIP(poll_val) & 1) { \ 1037 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ 1040 KMP_YIELD(TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ 1043 #endif // KMP_USE_FUTEX 1045 #else // KMP_USE_DYNAMIC_LOCK 1047 static kmp_user_lock_p
1048 __kmp_get_critical_section_ptr( kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid )
1050 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1056 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1058 if ( lck == NULL ) {
1063 lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section );
1064 __kmp_init_user_lock_with_checks( lck );
1065 __kmp_set_user_lock_location( lck, loc );
1067 __kmp_itt_critical_creating( lck );
1080 int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck );
1082 if ( status == 0 ) {
1085 __kmp_itt_critical_destroyed( lck );
1089 __kmp_destroy_user_lock_with_checks( lck );
1090 __kmp_user_lock_free( &idx, gtid, lck );
1091 lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1092 KMP_DEBUG_ASSERT( lck != NULL );
1098 #endif // KMP_USE_DYNAMIC_LOCK 1113 #if KMP_USE_DYNAMIC_LOCK 1114 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1117 kmp_user_lock_p lck;
1119 KC_TRACE( 10, (
"__kmpc_critical: called T#%d\n", global_tid ) );
1123 KMP_CHECK_USER_LOCK_INIT();
1125 if ( ( __kmp_user_lock_kind == lk_tas )
1126 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1127 lck = (kmp_user_lock_p)crit;
1129 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1130 else if ( ( __kmp_user_lock_kind == lk_futex )
1131 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1132 lck = (kmp_user_lock_p)crit;
1136 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
1139 if ( __kmp_env_consistency_check )
1140 __kmp_push_sync( global_tid, ct_critical, loc, lck );
1149 __kmp_itt_critical_acquiring( lck );
1152 __kmp_acquire_user_lock_with_checks( lck, global_tid );
1155 __kmp_itt_critical_acquired( lck );
1158 KA_TRACE( 15, (
"__kmpc_critical: done T#%d\n", global_tid ));
1159 #endif // KMP_USE_DYNAMIC_LOCK 1162 #if KMP_USE_DYNAMIC_LOCK 1165 static __forceinline kmp_dyna_lockseq_t
1166 __kmp_map_hint_to_lock(uintptr_t hint)
1169 # define KMP_TSX_LOCK(seq) lockseq_##seq 1171 # define KMP_TSX_LOCK(seq) __kmp_user_lock_seq 1174 if (hint & kmp_lock_hint_hle)
1175 return KMP_TSX_LOCK(hle);
1176 if (hint & kmp_lock_hint_rtm)
1177 return (__kmp_cpuinfo.rtm)? KMP_TSX_LOCK(rtm): __kmp_user_lock_seq;
1178 if (hint & kmp_lock_hint_adaptive)
1179 return (__kmp_cpuinfo.rtm)? KMP_TSX_LOCK(adaptive): __kmp_user_lock_seq;
1182 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1183 return __kmp_user_lock_seq;
1184 if ((hint & omp_lock_hint_speculative) && (hint & omp_lock_hint_nonspeculative))
1185 return __kmp_user_lock_seq;
1188 if (hint & omp_lock_hint_contended)
1189 return lockseq_queuing;
1192 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1196 if (hint & omp_lock_hint_speculative)
1197 return KMP_TSX_LOCK(hle);
1199 return __kmp_user_lock_seq;
1215 __kmpc_critical_with_hint(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit, uintptr_t hint )
1218 kmp_user_lock_p lck;
1220 KC_TRACE( 10, (
"__kmpc_critical: called T#%d\n", global_tid ) );
1222 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1225 kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
1226 if (KMP_IS_D_LOCK(lckseq)) {
1227 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(lckseq));
1229 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
1234 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1235 lck = (kmp_user_lock_p)lk;
1236 if (__kmp_env_consistency_check) {
1237 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint));
1240 __kmp_itt_critical_acquiring(lck);
1242 # if KMP_USE_INLINED_TAS 1243 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1244 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1246 # elif KMP_USE_INLINED_FUTEX 1247 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1248 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1252 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
1255 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1257 if (__kmp_env_consistency_check) {
1258 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint));
1261 __kmp_itt_critical_acquiring(lck);
1263 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
1267 __kmp_itt_critical_acquired( lck );
1270 KA_TRACE( 15, (
"__kmpc_critical: done T#%d\n", global_tid ));
1273 #endif // KMP_USE_DYNAMIC_LOCK 1287 kmp_user_lock_p lck;
1289 KC_TRACE( 10, (
"__kmpc_end_critical: called T#%d\n", global_tid ));
1291 #if KMP_USE_DYNAMIC_LOCK 1292 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
1293 lck = (kmp_user_lock_p)crit;
1294 KMP_ASSERT(lck != NULL);
1295 if (__kmp_env_consistency_check) {
1296 __kmp_pop_sync(global_tid, ct_critical, loc);
1299 __kmp_itt_critical_releasing( lck );
1301 # if KMP_USE_INLINED_TAS 1302 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1303 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1305 # elif KMP_USE_INLINED_FUTEX 1306 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1307 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1311 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1314 kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1315 KMP_ASSERT(ilk != NULL);
1317 if (__kmp_env_consistency_check) {
1318 __kmp_pop_sync(global_tid, ct_critical, loc);
1321 __kmp_itt_critical_releasing( lck );
1323 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1326 #else // KMP_USE_DYNAMIC_LOCK 1328 if ( ( __kmp_user_lock_kind == lk_tas )
1329 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1330 lck = (kmp_user_lock_p)crit;
1332 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1333 else if ( ( __kmp_user_lock_kind == lk_futex )
1334 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1335 lck = (kmp_user_lock_p)crit;
1339 lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit));
1342 KMP_ASSERT(lck != NULL);
1344 if ( __kmp_env_consistency_check )
1345 __kmp_pop_sync( global_tid, ct_critical, loc );
1348 __kmp_itt_critical_releasing( lck );
1351 __kmp_release_user_lock_with_checks( lck, global_tid );
1353 #if OMPT_SUPPORT && OMPT_BLAME 1355 ompt_callbacks.ompt_callback(ompt_event_release_critical)) {
1356 ompt_callbacks.ompt_callback(ompt_event_release_critical)(
1361 #endif // KMP_USE_DYNAMIC_LOCK 1363 KA_TRACE( 15, (
"__kmpc_end_critical: done T#%d\n", global_tid ));
1379 KC_TRACE( 10, (
"__kmpc_barrier_master: called T#%d\n", global_tid ) );
1381 if (! TCR_4(__kmp_init_parallel))
1382 __kmp_parallel_initialize();
1384 if ( __kmp_env_consistency_check )
1385 __kmp_check_barrier( global_tid, ct_barrier, loc );
1388 __kmp_threads[global_tid]->th.th_ident = loc;
1390 status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL );
1392 return (status != 0) ? 0 : 1;
1407 KC_TRACE( 10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid ));
1409 __kmp_end_split_barrier ( bs_plain_barrier, global_tid );
1427 KC_TRACE( 10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid ));
1429 if (! TCR_4(__kmp_init_parallel))
1430 __kmp_parallel_initialize();
1432 if ( __kmp_env_consistency_check ) {
1434 KMP_WARNING( ConstructIdentInvalid );
1436 __kmp_check_barrier( global_tid, ct_barrier, loc );
1440 __kmp_threads[global_tid]->th.th_ident = loc;
1442 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
1446 if ( __kmp_env_consistency_check ) {
1450 if ( global_tid < 0 ) {
1451 KMP_WARNING( ThreadIdentInvalid );
1457 __kmp_pop_sync( global_tid, ct_master, loc );
1480 kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
1485 #if OMPT_SUPPORT && OMPT_TRACE 1486 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1487 kmp_team_t *team = this_thr -> th.th_team;
1488 int tid = __kmp_tid_from_gtid( global_tid );
1492 if (ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)) {
1493 ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)(
1494 team->t.ompt_team_info.parallel_id,
1495 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id,
1496 team->t.ompt_team_info.microtask);
1499 if (ompt_callbacks.ompt_callback(ompt_event_single_others_begin)) {
1500 ompt_callbacks.ompt_callback(ompt_event_single_others_begin)(
1501 team->t.ompt_team_info.parallel_id,
1502 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1504 this_thr->th.ompt_thread_info.state = ompt_state_wait_single;
1524 __kmp_exit_single( global_tid );
1527 #if OMPT_SUPPORT && OMPT_TRACE 1528 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1529 kmp_team_t *team = this_thr -> th.th_team;
1530 int tid = __kmp_tid_from_gtid( global_tid );
1533 ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)) {
1534 ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)(
1535 team->t.ompt_team_info.parallel_id,
1536 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1551 KE_TRACE( 10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1553 #if OMPT_SUPPORT && OMPT_TRACE 1555 ompt_callbacks.ompt_callback(ompt_event_loop_end)) {
1556 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1557 kmp_team_t *team = this_thr -> th.th_team;
1558 int tid = __kmp_tid_from_gtid( global_tid );
1560 ompt_callbacks.ompt_callback(ompt_event_loop_end)(
1561 team->t.ompt_team_info.parallel_id,
1562 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1566 if ( __kmp_env_consistency_check )
1567 __kmp_pop_workshare( global_tid, ct_pdo, loc );
1576 ompc_set_num_threads(
int arg )
1579 __kmp_set_num_threads( arg, __kmp_entry_gtid() );
1583 ompc_set_dynamic(
int flag )
1588 thread = __kmp_entry_thread();
1590 __kmp_save_internal_controls( thread );
1592 set__dynamic( thread, flag ? TRUE : FALSE );
1596 ompc_set_nested(
int flag )
1601 thread = __kmp_entry_thread();
1603 __kmp_save_internal_controls( thread );
1605 set__nested( thread, flag ? TRUE : FALSE );
1609 ompc_set_max_active_levels(
int max_active_levels )
1615 __kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels );
1619 ompc_set_schedule( omp_sched_t kind,
int modifier )
1622 __kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier );
1626 ompc_get_ancestor_thread_num(
int level )
1628 return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level );
1632 ompc_get_team_size(
int level )
1634 return __kmp_get_team_size( __kmp_entry_gtid(), level );
1638 kmpc_set_stacksize(
int arg )
1641 __kmp_aux_set_stacksize( arg );
1645 kmpc_set_stacksize_s(
size_t arg )
1648 __kmp_aux_set_stacksize( arg );
1652 kmpc_set_blocktime(
int arg )
1657 gtid = __kmp_entry_gtid();
1658 tid = __kmp_tid_from_gtid(gtid);
1659 thread = __kmp_thread_from_gtid(gtid);
1661 __kmp_aux_set_blocktime( arg, thread, tid );
1665 kmpc_set_library(
int arg )
1668 __kmp_user_set_library( (
enum library_type)arg );
1672 kmpc_set_defaults(
char const * str )
1675 __kmp_aux_set_defaults( str, KMP_STRLEN( str ) );
1679 kmpc_set_affinity_mask_proc(
int proc,
void **mask )
1681 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1684 if ( ! TCR_4(__kmp_init_middle) ) {
1685 __kmp_middle_initialize();
1687 return __kmp_aux_set_affinity_mask_proc( proc, mask );
1692 kmpc_unset_affinity_mask_proc(
int proc,
void **mask )
1694 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1697 if ( ! TCR_4(__kmp_init_middle) ) {
1698 __kmp_middle_initialize();
1700 return __kmp_aux_unset_affinity_mask_proc( proc, mask );
1705 kmpc_get_affinity_mask_proc(
int proc,
void **mask )
1707 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED 1710 if ( ! TCR_4(__kmp_init_middle) ) {
1711 __kmp_middle_initialize();
1713 return __kmp_aux_get_affinity_mask_proc( proc, mask );
1764 KC_TRACE( 10, (
"__kmpc_copyprivate: called T#%d\n", gtid ));
1768 data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data;
1770 if ( __kmp_env_consistency_check ) {
1772 KMP_WARNING( ConstructIdentInvalid );
1778 if (didit) *data_ptr = cpy_data;
1782 __kmp_threads[gtid]->th.th_ident = loc;
1784 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1786 if (! didit) (*cpy_func)( cpy_data, *data_ptr );
1792 __kmp_threads[gtid]->th.th_ident = loc;
1794 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1799 #define INIT_LOCK __kmp_init_user_lock_with_checks 1800 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks 1801 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks 1802 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed 1803 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks 1804 #define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed 1805 #define RELEASE_LOCK __kmp_release_user_lock_with_checks 1806 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks 1807 #define TEST_LOCK __kmp_test_user_lock_with_checks 1808 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks 1809 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks 1810 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks 1818 #if KMP_USE_DYNAMIC_LOCK 1821 static __forceinline
void 1822 __kmp_init_lock_with_hint(
ident_t *loc,
void **lock, kmp_dyna_lockseq_t seq)
1824 if (KMP_IS_D_LOCK(seq)) {
1825 KMP_INIT_D_LOCK(lock, seq);
1827 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
1830 KMP_INIT_I_LOCK(lock, seq);
1832 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
1833 __kmp_itt_lock_creating(ilk->lock, loc);
1839 static __forceinline
void 1840 __kmp_init_nest_lock_with_hint(
ident_t *loc,
void **lock, kmp_dyna_lockseq_t seq)
1844 if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
1845 seq = __kmp_user_lock_seq;
1849 seq = lockseq_nested_tas;
1853 seq = lockseq_nested_futex;
1856 case lockseq_ticket:
1857 seq = lockseq_nested_ticket;
1859 case lockseq_queuing:
1860 seq = lockseq_nested_queuing;
1863 seq = lockseq_nested_drdpa;
1866 seq = lockseq_nested_queuing;
1868 KMP_INIT_I_LOCK(lock, seq);
1870 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
1871 __kmp_itt_lock_creating(ilk->lock, loc);
1877 __kmpc_init_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
void **user_lock, uintptr_t hint)
1879 KMP_DEBUG_ASSERT(__kmp_init_serial);
1880 if (__kmp_env_consistency_check && user_lock == NULL) {
1881 KMP_FATAL(LockIsUninitialized,
"omp_init_lock_with_hint");
1884 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
1889 __kmpc_init_nest_lock_with_hint(
ident_t *loc, kmp_int32 gtid,
void **user_lock, uintptr_t hint)
1891 KMP_DEBUG_ASSERT(__kmp_init_serial);
1892 if (__kmp_env_consistency_check && user_lock == NULL) {
1893 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock_with_hint");
1896 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
1899 #endif // KMP_USE_DYNAMIC_LOCK 1903 __kmpc_init_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1904 #if KMP_USE_DYNAMIC_LOCK 1905 KMP_DEBUG_ASSERT(__kmp_init_serial);
1906 if (__kmp_env_consistency_check && user_lock == NULL) {
1907 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
1909 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
1911 #else // KMP_USE_DYNAMIC_LOCK 1913 static char const *
const func =
"omp_init_lock";
1914 kmp_user_lock_p lck;
1915 KMP_DEBUG_ASSERT( __kmp_init_serial );
1917 if ( __kmp_env_consistency_check ) {
1918 if ( user_lock == NULL ) {
1919 KMP_FATAL( LockIsUninitialized, func );
1923 KMP_CHECK_USER_LOCK_INIT();
1925 if ( ( __kmp_user_lock_kind == lk_tas )
1926 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1927 lck = (kmp_user_lock_p)user_lock;
1929 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1930 else if ( ( __kmp_user_lock_kind == lk_futex )
1931 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1932 lck = (kmp_user_lock_p)user_lock;
1936 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1939 __kmp_set_user_lock_location( lck, loc );
1941 #if OMPT_SUPPORT && OMPT_TRACE 1943 ompt_callbacks.ompt_callback(ompt_event_init_lock)) {
1944 ompt_callbacks.ompt_callback(ompt_event_init_lock)((uint64_t) lck);
1949 __kmp_itt_lock_creating( lck );
1952 #endif // KMP_USE_DYNAMIC_LOCK 1957 __kmpc_init_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1958 #if KMP_USE_DYNAMIC_LOCK 1960 KMP_DEBUG_ASSERT(__kmp_init_serial);
1961 if (__kmp_env_consistency_check && user_lock == NULL) {
1962 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
1964 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
1966 #else // KMP_USE_DYNAMIC_LOCK 1968 static char const *
const func =
"omp_init_nest_lock";
1969 kmp_user_lock_p lck;
1970 KMP_DEBUG_ASSERT( __kmp_init_serial );
1972 if ( __kmp_env_consistency_check ) {
1973 if ( user_lock == NULL ) {
1974 KMP_FATAL( LockIsUninitialized, func );
1978 KMP_CHECK_USER_LOCK_INIT();
1980 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1981 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1982 lck = (kmp_user_lock_p)user_lock;
1984 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1985 else if ( ( __kmp_user_lock_kind == lk_futex )
1986 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1987 <= OMP_NEST_LOCK_T_SIZE ) ) {
1988 lck = (kmp_user_lock_p)user_lock;
1992 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1995 INIT_NESTED_LOCK( lck );
1996 __kmp_set_user_lock_location( lck, loc );
1998 #if OMPT_SUPPORT && OMPT_TRACE 2000 ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)) {
2001 ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)((uint64_t) lck);
2006 __kmp_itt_lock_creating( lck );
2009 #endif // KMP_USE_DYNAMIC_LOCK 2013 __kmpc_destroy_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
2014 #if KMP_USE_DYNAMIC_LOCK 2017 kmp_user_lock_p lck;
2018 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2019 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2021 lck = (kmp_user_lock_p)user_lock;
2023 __kmp_itt_lock_destroyed(lck);
2025 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2027 kmp_user_lock_p lck;
2029 if ( ( __kmp_user_lock_kind == lk_tas )
2030 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2031 lck = (kmp_user_lock_p)user_lock;
2033 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2034 else if ( ( __kmp_user_lock_kind == lk_futex )
2035 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2036 lck = (kmp_user_lock_p)user_lock;
2040 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_lock" );
2043 #if OMPT_SUPPORT && OMPT_TRACE 2045 ompt_callbacks.ompt_callback(ompt_event_destroy_lock)) {
2046 ompt_callbacks.ompt_callback(ompt_event_destroy_lock)((uint64_t) lck);
2051 __kmp_itt_lock_destroyed( lck );
2053 DESTROY_LOCK( lck );
2055 if ( ( __kmp_user_lock_kind == lk_tas )
2056 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2059 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2060 else if ( ( __kmp_user_lock_kind == lk_futex )
2061 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2066 __kmp_user_lock_free( user_lock, gtid, lck );
2068 #endif // KMP_USE_DYNAMIC_LOCK 2073 __kmpc_destroy_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
2074 #if KMP_USE_DYNAMIC_LOCK 2077 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2078 __kmp_itt_lock_destroyed(ilk->lock);
2080 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2082 #else // KMP_USE_DYNAMIC_LOCK 2084 kmp_user_lock_p lck;
2086 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2087 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2088 lck = (kmp_user_lock_p)user_lock;
2090 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2091 else if ( ( __kmp_user_lock_kind == lk_futex )
2092 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2093 <= OMP_NEST_LOCK_T_SIZE ) ) {
2094 lck = (kmp_user_lock_p)user_lock;
2098 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_nest_lock" );
2101 #if OMPT_SUPPORT && OMPT_TRACE 2103 ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)) {
2104 ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)((uint64_t) lck);
2109 __kmp_itt_lock_destroyed( lck );
2112 DESTROY_NESTED_LOCK( lck );
2114 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2115 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2118 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2119 else if ( ( __kmp_user_lock_kind == lk_futex )
2120 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2121 <= OMP_NEST_LOCK_T_SIZE ) ) {
2126 __kmp_user_lock_free( user_lock, gtid, lck );
2128 #endif // KMP_USE_DYNAMIC_LOCK 2132 __kmpc_set_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
2134 #if KMP_USE_DYNAMIC_LOCK 2135 int tag = KMP_EXTRACT_D_TAG(user_lock);
2137 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2139 # if KMP_USE_INLINED_TAS 2140 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2141 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2143 # elif KMP_USE_INLINED_FUTEX 2144 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2145 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2149 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2152 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2155 #else // KMP_USE_DYNAMIC_LOCK 2157 kmp_user_lock_p lck;
2159 if ( ( __kmp_user_lock_kind == lk_tas )
2160 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2161 lck = (kmp_user_lock_p)user_lock;
2163 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2164 else if ( ( __kmp_user_lock_kind == lk_futex )
2165 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2166 lck = (kmp_user_lock_p)user_lock;
2170 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_lock" );
2174 __kmp_itt_lock_acquiring( lck );
2177 ACQUIRE_LOCK( lck, gtid );
2180 __kmp_itt_lock_acquired( lck );
2183 #if OMPT_SUPPORT && OMPT_TRACE 2185 ompt_callbacks.ompt_callback(ompt_event_acquired_lock)) {
2186 ompt_callbacks.ompt_callback(ompt_event_acquired_lock)((uint64_t) lck);
2190 #endif // KMP_USE_DYNAMIC_LOCK 2194 __kmpc_set_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
2195 #if KMP_USE_DYNAMIC_LOCK 2198 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2200 KMP_D_LOCK_FUNC(user_lock,
set)((kmp_dyna_lock_t *)user_lock, gtid);
2202 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2205 #if OMPT_SUPPORT && OMPT_TRACE 2211 #else // KMP_USE_DYNAMIC_LOCK 2213 kmp_user_lock_p lck;
2215 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2216 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2217 lck = (kmp_user_lock_p)user_lock;
2219 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2220 else if ( ( __kmp_user_lock_kind == lk_futex )
2221 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2222 <= OMP_NEST_LOCK_T_SIZE ) ) {
2223 lck = (kmp_user_lock_p)user_lock;
2227 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_nest_lock" );
2231 __kmp_itt_lock_acquiring( lck );
2234 ACQUIRE_NESTED_LOCK( lck, gtid, &acquire_status );
2237 __kmp_itt_lock_acquired( lck );
2240 #if OMPT_SUPPORT && OMPT_TRACE 2242 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2243 if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first))
2244 ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first)((uint64_t) lck);
2246 if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next))
2247 ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next)((uint64_t) lck);
2252 #endif // KMP_USE_DYNAMIC_LOCK 2256 __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2258 #if KMP_USE_DYNAMIC_LOCK 2260 int tag = KMP_EXTRACT_D_TAG(user_lock);
2262 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2264 # if KMP_USE_INLINED_TAS 2265 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2266 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
2268 # elif KMP_USE_INLINED_FUTEX 2269 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2270 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
2274 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2277 #else // KMP_USE_DYNAMIC_LOCK 2279 kmp_user_lock_p lck;
2284 if ( ( __kmp_user_lock_kind == lk_tas )
2285 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2286 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2289 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
2291 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2295 lck = (kmp_user_lock_p)user_lock;
2298 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2299 else if ( ( __kmp_user_lock_kind == lk_futex )
2300 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2301 lck = (kmp_user_lock_p)user_lock;
2305 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_lock" );
2309 __kmp_itt_lock_releasing( lck );
2312 RELEASE_LOCK( lck, gtid );
2314 #if OMPT_SUPPORT && OMPT_BLAME 2316 ompt_callbacks.ompt_callback(ompt_event_release_lock)) {
2317 ompt_callbacks.ompt_callback(ompt_event_release_lock)((uint64_t) lck);
2321 #endif // KMP_USE_DYNAMIC_LOCK 2326 __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2328 #if KMP_USE_DYNAMIC_LOCK 2331 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2333 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2335 #else // KMP_USE_DYNAMIC_LOCK 2337 kmp_user_lock_p lck;
2341 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2342 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2343 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2345 kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;
2347 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
2349 if ( --(tl->lk.depth_locked) == 0 ) {
2350 TCW_4(tl->lk.poll, 0);
2355 lck = (kmp_user_lock_p)user_lock;
2358 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2359 else if ( ( __kmp_user_lock_kind == lk_futex )
2360 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2361 <= OMP_NEST_LOCK_T_SIZE ) ) {
2362 lck = (kmp_user_lock_p)user_lock;
2366 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_nest_lock" );
2370 __kmp_itt_lock_releasing( lck );
2374 release_status = RELEASE_NESTED_LOCK( lck, gtid );
2375 #if OMPT_SUPPORT && OMPT_BLAME 2377 if (release_status == KMP_LOCK_RELEASED) {
2378 if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)) {
2379 ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)(
2382 }
else if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)) {
2383 ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)(
2389 #endif // KMP_USE_DYNAMIC_LOCK 2394 __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2398 #if KMP_USE_DYNAMIC_LOCK 2400 int tag = KMP_EXTRACT_D_TAG(user_lock);
2402 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2404 # if KMP_USE_INLINED_TAS 2405 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2406 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
2408 # elif KMP_USE_INLINED_FUTEX 2409 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2410 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
2414 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2418 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2423 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2428 #else // KMP_USE_DYNAMIC_LOCK 2430 kmp_user_lock_p lck;
2433 if ( ( __kmp_user_lock_kind == lk_tas )
2434 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2435 lck = (kmp_user_lock_p)user_lock;
2437 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2438 else if ( ( __kmp_user_lock_kind == lk_futex )
2439 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2440 lck = (kmp_user_lock_p)user_lock;
2444 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_lock" );
2448 __kmp_itt_lock_acquiring( lck );
2451 rc = TEST_LOCK( lck, gtid );
2454 __kmp_itt_lock_acquired( lck );
2456 __kmp_itt_lock_cancelled( lck );
2459 return ( rc ? FTN_TRUE : FTN_FALSE );
2463 #endif // KMP_USE_DYNAMIC_LOCK 2468 __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2470 #if KMP_USE_DYNAMIC_LOCK 2473 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2475 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
2478 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2480 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2485 #else // KMP_USE_DYNAMIC_LOCK 2487 kmp_user_lock_p lck;
2490 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2491 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2492 lck = (kmp_user_lock_p)user_lock;
2494 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 2495 else if ( ( __kmp_user_lock_kind == lk_futex )
2496 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2497 <= OMP_NEST_LOCK_T_SIZE ) ) {
2498 lck = (kmp_user_lock_p)user_lock;
2502 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_nest_lock" );
2506 __kmp_itt_lock_acquiring( lck );
2509 rc = TEST_NESTED_LOCK( lck, gtid );
2512 __kmp_itt_lock_acquired( lck );
2514 __kmp_itt_lock_cancelled( lck );
2521 #endif // KMP_USE_DYNAMIC_LOCK 2534 #define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \ 2535 ( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) ) 2537 #define __KMP_GET_REDUCTION_METHOD(gtid) \ 2538 ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) 2544 static __forceinline
void 2545 __kmp_enter_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2551 kmp_user_lock_p lck;
2553 #if KMP_USE_DYNAMIC_LOCK 2555 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
2558 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
2559 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(__kmp_user_lock_seq));
2561 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(__kmp_user_lock_seq));
2566 if (KMP_EXTRACT_D_TAG(lk) != 0) {
2567 lck = (kmp_user_lock_p)lk;
2568 KMP_DEBUG_ASSERT(lck != NULL);
2569 if (__kmp_env_consistency_check) {
2570 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
2572 KMP_D_LOCK_FUNC(lk,
set)(lk, global_tid);
2574 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
2576 KMP_DEBUG_ASSERT(lck != NULL);
2577 if (__kmp_env_consistency_check) {
2578 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
2580 KMP_I_LOCK_FUNC(ilk,
set)(lck, global_tid);
2583 #else // KMP_USE_DYNAMIC_LOCK 2588 if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) {
2589 lck = (kmp_user_lock_p)crit;
2592 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
2594 KMP_DEBUG_ASSERT( lck != NULL );
2596 if ( __kmp_env_consistency_check )
2597 __kmp_push_sync( global_tid, ct_critical, loc, lck );
2599 __kmp_acquire_user_lock_with_checks( lck, global_tid );
2601 #endif // KMP_USE_DYNAMIC_LOCK 2605 static __forceinline
void 2606 __kmp_end_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2608 kmp_user_lock_p lck;
2610 #if KMP_USE_DYNAMIC_LOCK 2612 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
2613 lck = (kmp_user_lock_p)crit;
2614 if (__kmp_env_consistency_check)
2615 __kmp_pop_sync(global_tid, ct_critical, loc);
2616 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
2618 kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
2619 if (__kmp_env_consistency_check)
2620 __kmp_pop_sync(global_tid, ct_critical, loc);
2621 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
2624 #else // KMP_USE_DYNAMIC_LOCK 2628 if ( __kmp_base_user_lock_size > 32 ) {
2629 lck = *( (kmp_user_lock_p *) crit );
2630 KMP_ASSERT( lck != NULL );
2632 lck = (kmp_user_lock_p) crit;
2635 if ( __kmp_env_consistency_check )
2636 __kmp_pop_sync( global_tid, ct_critical, loc );
2638 __kmp_release_user_lock_with_checks( lck, global_tid );
2640 #endif // KMP_USE_DYNAMIC_LOCK 2660 ident_t *loc, kmp_int32 global_tid,
2661 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2662 kmp_critical_name *lck ) {
2666 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2670 int teams_swapped = 0, task_state;
2672 KA_TRACE( 10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) );
2679 if( ! TCR_4( __kmp_init_parallel ) )
2680 __kmp_parallel_initialize();
2683 #if KMP_USE_DYNAMIC_LOCK 2684 if ( __kmp_env_consistency_check )
2685 __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
2687 if ( __kmp_env_consistency_check )
2688 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2692 th = __kmp_thread_from_gtid(global_tid);
2693 if( th->th.th_teams_microtask ) {
2694 team = th->th.th_team;
2695 if( team->t.t_level == th->th.th_teams_level ) {
2697 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
2700 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2701 th->th.th_team = team->t.t_parent;
2702 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
2703 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
2704 task_state = th->th.th_task_state;
2705 th->th.th_task_state = 0;
2708 #endif // OMP_40_ENABLED 2719 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2720 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2722 if( packed_reduction_method == critical_reduce_block ) {
2724 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2727 }
else if( packed_reduction_method == empty_reduce_block ) {
2732 }
else if( packed_reduction_method == atomic_reduce_block ) {
2739 if ( __kmp_env_consistency_check )
2740 __kmp_pop_sync( global_tid, ct_reduce, loc );
2742 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2755 __kmp_threads[global_tid]->th.th_ident = loc;
2757 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func );
2758 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2762 if ( __kmp_env_consistency_check ) {
2764 __kmp_pop_sync( global_tid, ct_reduce, loc );
2775 if( teams_swapped ) {
2777 th->th.th_info.ds.ds_tid = 0;
2778 th->th.th_team = team;
2779 th->th.th_team_nproc = team->t.t_nproc;
2780 th->th.th_task_team = team->t.t_task_team[task_state];
2781 th->th.th_task_state = task_state;
2784 KA_TRACE( 10, (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2800 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2802 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) );
2804 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2806 if( packed_reduction_method == critical_reduce_block ) {
2808 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2810 }
else if( packed_reduction_method == empty_reduce_block ) {
2814 }
else if( packed_reduction_method == atomic_reduce_block ) {
2821 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2832 if ( __kmp_env_consistency_check )
2833 __kmp_pop_sync( global_tid, ct_reduce, loc );
2835 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2857 ident_t *loc, kmp_int32 global_tid,
2858 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
2859 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2860 kmp_critical_name *lck )
2864 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2866 KA_TRACE( 10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid ) );
2873 if( ! TCR_4( __kmp_init_parallel ) )
2874 __kmp_parallel_initialize();
2877 #if KMP_USE_DYNAMIC_LOCK 2878 if ( __kmp_env_consistency_check )
2879 __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
2881 if ( __kmp_env_consistency_check )
2882 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2885 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2886 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2888 if( packed_reduction_method == critical_reduce_block ) {
2890 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2893 }
else if( packed_reduction_method == empty_reduce_block ) {
2898 }
else if( packed_reduction_method == atomic_reduce_block ) {
2902 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2908 __kmp_threads[global_tid]->th.th_ident = loc;
2910 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func );
2911 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2915 if ( __kmp_env_consistency_check ) {
2917 __kmp_pop_sync( global_tid, ct_reduce, loc );
2928 KA_TRACE( 10, (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2945 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2947 KA_TRACE( 10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid ) );
2949 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2954 if( packed_reduction_method == critical_reduce_block ) {
2956 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2960 __kmp_threads[global_tid]->th.th_ident = loc;
2962 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2964 }
else if( packed_reduction_method == empty_reduce_block ) {
2970 __kmp_threads[global_tid]->th.th_ident = loc;
2972 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2974 }
else if( packed_reduction_method == atomic_reduce_block ) {
2978 __kmp_threads[global_tid]->th.th_ident = loc;
2980 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2982 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2985 __kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid );
2994 if ( __kmp_env_consistency_check )
2995 __kmp_pop_sync( global_tid, ct_reduce, loc );
2997 KA_TRACE( 10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
3002 #undef __KMP_GET_REDUCTION_METHOD 3003 #undef __KMP_SET_REDUCTION_METHOD 3008 __kmpc_get_taskid() {
3011 kmp_info_t * thread;
3013 gtid = __kmp_get_gtid();
3017 thread = __kmp_thread_from_gtid( gtid );
3018 return thread->th.th_current_task->td_task_id;
3024 __kmpc_get_parent_taskid() {
3027 kmp_info_t * thread;
3028 kmp_taskdata_t * parent_task;
3030 gtid = __kmp_get_gtid();
3034 thread = __kmp_thread_from_gtid( gtid );
3035 parent_task = thread->th.th_current_task->td_parent;
3036 return ( parent_task == NULL ? 0 : parent_task->td_task_id );
3040 void __kmpc_place_threads(
int nS,
int sO,
int nC,
int cO,
int nT)
3042 if ( ! __kmp_init_serial ) {
3043 __kmp_serial_initialize();
3045 __kmp_place_num_sockets = nS;
3046 __kmp_place_socket_offset = sO;
3047 __kmp_place_num_cores = nC;
3048 __kmp_place_core_offset = cO;
3049 __kmp_place_num_threads_per_core = nT;
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_STOP_EXPLICIT_TIMER(name)
"Stops" an explicit timer.
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
#define KMP_IDENT_AUTOPAR
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)