23 #include "kmp_debug.h" 35 #define KMP_PAD(type, sz) (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1)) 36 #define KMP_GTID_DNE (-2) 57 #if KMP_OS_LINUX && defined(KMP_GOMP_COMPAT) 58 # define OMP_LOCK_T_SIZE sizeof(int) 59 # define OMP_NEST_LOCK_T_SIZE sizeof(void *) 61 # define OMP_LOCK_T_SIZE sizeof(void *) 62 # define OMP_NEST_LOCK_T_SIZE sizeof(void *) 70 #define OMP_CRITICAL_SIZE sizeof(void *) 71 #define INTEL_CRITICAL_SIZE 32 76 typedef kmp_uint32 kmp_lock_flags_t;
78 #define kmp_lf_critical_section 1 83 typedef kmp_uint32 kmp_lock_index_t;
89 struct kmp_lock_pool {
90 union kmp_user_lock *next;
91 kmp_lock_index_t index;
94 typedef struct kmp_lock_pool kmp_lock_pool_t;
97 extern void __kmp_validate_locks(
void );
141 struct kmp_base_tas_lock {
142 volatile kmp_int32 poll;
144 kmp_int32 depth_locked;
147 typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
150 kmp_base_tas_lock_t lk;
151 kmp_lock_pool_t pool;
156 typedef union kmp_tas_lock kmp_tas_lock_t;
162 #define KMP_TAS_LOCK_INITIALIZER( lock ) { { 0, 0 } } 164 extern int __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
165 extern int __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
166 extern int __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
167 extern void __kmp_init_tas_lock( kmp_tas_lock_t *lck );
168 extern void __kmp_destroy_tas_lock( kmp_tas_lock_t *lck );
170 extern int __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
171 extern int __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
172 extern int __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
173 extern void __kmp_init_nested_tas_lock( kmp_tas_lock_t *lck );
174 extern void __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck );
176 #define KMP_LOCK_RELEASED 1 177 #define KMP_LOCK_STILL_HELD 0 178 #define KMP_LOCK_ACQUIRED_FIRST 1 179 #define KMP_LOCK_ACQUIRED_NEXT 0 181 #define KMP_USE_FUTEX (KMP_OS_LINUX && !KMP_OS_CNK && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)) 197 struct kmp_base_futex_lock {
198 volatile kmp_int32 poll;
201 kmp_int32 depth_locked;
204 typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
206 union kmp_futex_lock {
207 kmp_base_futex_lock_t lk;
208 kmp_lock_pool_t pool;
213 typedef union kmp_futex_lock kmp_futex_lock_t;
219 #define KMP_FUTEX_LOCK_INITIALIZER( lock ) { { 0, 0 } } 221 extern int __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
222 extern int __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
223 extern int __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
224 extern void __kmp_init_futex_lock( kmp_futex_lock_t *lck );
225 extern void __kmp_destroy_futex_lock( kmp_futex_lock_t *lck );
227 extern int __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
228 extern int __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
229 extern int __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
230 extern void __kmp_init_nested_futex_lock( kmp_futex_lock_t *lck );
231 extern void __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck );
233 #endif // KMP_USE_FUTEX 245 struct kmp_base_ticket_lock {
247 std::atomic_bool initialized;
248 volatile union kmp_ticket_lock *
self;
250 std::atomic_uint next_ticket;
251 std::atomic_uint now_serving;
252 std::atomic_int owner_id;
253 std::atomic_int depth_locked;
254 kmp_lock_flags_t flags;
257 struct kmp_base_ticket_lock {
259 std::atomic<bool> initialized;
260 volatile union kmp_ticket_lock *
self;
262 std::atomic<unsigned> next_ticket;
263 std::atomic<unsigned> now_serving;
264 std::atomic<int> owner_id;
265 std::atomic<int> depth_locked;
266 kmp_lock_flags_t flags;
272 struct kmp_base_ticket_lock;
274 #endif // !__cplusplus 276 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
278 union KMP_ALIGN_CACHE kmp_ticket_lock {
279 kmp_base_ticket_lock_t lk;
280 kmp_lock_pool_t pool;
282 char lk_pad[ KMP_PAD( kmp_base_ticket_lock_t, CACHE_LINE ) ];
285 typedef union kmp_ticket_lock kmp_ticket_lock_t;
292 #define KMP_TICKET_LOCK_INITIALIZER( lock ) { { ATOMIC_VAR_INIT(true), \ 295 ATOMIC_VAR_INIT(0U), \ 296 ATOMIC_VAR_INIT(0U), \ 297 ATOMIC_VAR_INIT(0), \ 298 ATOMIC_VAR_INIT(-1) } } 300 extern int __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
301 extern int __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
302 extern int __kmp_test_ticket_lock_with_cheks( kmp_ticket_lock_t *lck, kmp_int32 gtid );
303 extern int __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
304 extern void __kmp_init_ticket_lock( kmp_ticket_lock_t *lck );
305 extern void __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck );
307 extern int __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
308 extern int __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
309 extern int __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
310 extern void __kmp_init_nested_ticket_lock( kmp_ticket_lock_t *lck );
311 extern void __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck );
318 #if KMP_USE_ADAPTIVE_LOCKS 320 struct kmp_adaptive_lock_info;
322 typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
324 #if KMP_DEBUG_ADAPTIVE_LOCKS 326 struct kmp_adaptive_lock_statistics {
328 kmp_adaptive_lock_info_t * next;
329 kmp_adaptive_lock_info_t * prev;
332 kmp_uint32 successfulSpeculations;
333 kmp_uint32 hardFailedSpeculations;
334 kmp_uint32 softFailedSpeculations;
335 kmp_uint32 nonSpeculativeAcquires;
336 kmp_uint32 nonSpeculativeAcquireAttempts;
337 kmp_uint32 lemmingYields;
340 typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
342 extern void __kmp_print_speculative_stats();
343 extern void __kmp_init_speculative_stats();
345 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 347 struct kmp_adaptive_lock_info
354 kmp_uint32
volatile badness;
355 kmp_uint32
volatile acquire_attempts;
357 kmp_uint32 max_badness;
358 kmp_uint32 max_soft_retries;
360 #if KMP_DEBUG_ADAPTIVE_LOCKS 361 kmp_adaptive_lock_statistics_t
volatile stats;
365 #endif // KMP_USE_ADAPTIVE_LOCKS 368 struct kmp_base_queuing_lock {
371 volatile union kmp_queuing_lock *initialized;
377 volatile kmp_int32 tail_id;
379 volatile kmp_int32 head_id;
382 volatile kmp_uint32 next_ticket;
383 volatile kmp_uint32 now_serving;
384 volatile kmp_int32 owner_id;
385 kmp_int32 depth_locked;
387 kmp_lock_flags_t flags;
390 typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
392 KMP_BUILD_ASSERT( offsetof( kmp_base_queuing_lock_t, tail_id ) % 8 == 0 );
394 union KMP_ALIGN_CACHE kmp_queuing_lock {
395 kmp_base_queuing_lock_t lk;
396 kmp_lock_pool_t pool;
398 char lk_pad[ KMP_PAD( kmp_base_queuing_lock_t, CACHE_LINE ) ];
401 typedef union kmp_queuing_lock kmp_queuing_lock_t;
403 extern int __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
404 extern int __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
405 extern int __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
406 extern void __kmp_init_queuing_lock( kmp_queuing_lock_t *lck );
407 extern void __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck );
409 extern int __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
410 extern int __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
411 extern int __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
412 extern void __kmp_init_nested_queuing_lock( kmp_queuing_lock_t *lck );
413 extern void __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck );
415 #if KMP_USE_ADAPTIVE_LOCKS 420 struct kmp_base_adaptive_lock {
421 kmp_base_queuing_lock qlk;
422 KMP_ALIGN(CACHE_LINE)
423 kmp_adaptive_lock_info_t adaptive;
426 typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
428 union KMP_ALIGN_CACHE kmp_adaptive_lock {
429 kmp_base_adaptive_lock_t lk;
430 kmp_lock_pool_t pool;
432 char lk_pad[ KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE) ];
434 typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
436 # define GET_QLK_PTR(l) ((kmp_queuing_lock_t *) & (l)->lk.qlk) 438 #endif // KMP_USE_ADAPTIVE_LOCKS 444 struct kmp_base_drdpa_lock {
455 volatile union kmp_drdpa_lock * initialized;
457 volatile struct kmp_lock_poll {
460 volatile kmp_uint64 mask;
461 kmp_uint64 cleanup_ticket;
462 volatile struct kmp_lock_poll * old_polls;
463 kmp_uint32 num_polls;
471 volatile kmp_uint64 next_ticket;
490 kmp_uint64 now_serving;
491 volatile kmp_uint32 owner_id;
492 kmp_int32 depth_locked;
493 kmp_lock_flags_t flags;
496 typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
498 union KMP_ALIGN_CACHE kmp_drdpa_lock {
499 kmp_base_drdpa_lock_t lk;
500 kmp_lock_pool_t pool;
502 char lk_pad[ KMP_PAD( kmp_base_drdpa_lock_t, CACHE_LINE ) ];
505 typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
507 extern int __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
508 extern int __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
509 extern int __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
510 extern void __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck );
511 extern void __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck );
513 extern int __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
514 extern int __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
515 extern int __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
516 extern void __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
517 extern void __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
535 typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
537 #define KMP_BOOTSTRAP_LOCK_INITIALIZER( lock ) KMP_TICKET_LOCK_INITIALIZER( (lock) ) 540 __kmp_acquire_bootstrap_lock( kmp_bootstrap_lock_t *lck )
542 return __kmp_acquire_ticket_lock( lck, KMP_GTID_DNE );
546 __kmp_test_bootstrap_lock( kmp_bootstrap_lock_t *lck )
548 return __kmp_test_ticket_lock( lck, KMP_GTID_DNE );
552 __kmp_release_bootstrap_lock( kmp_bootstrap_lock_t *lck )
554 __kmp_release_ticket_lock( lck, KMP_GTID_DNE );
558 __kmp_init_bootstrap_lock( kmp_bootstrap_lock_t *lck )
560 __kmp_init_ticket_lock( lck );
564 __kmp_destroy_bootstrap_lock( kmp_bootstrap_lock_t *lck )
566 __kmp_destroy_ticket_lock( lck );
582 typedef kmp_ticket_lock_t kmp_lock_t;
585 __kmp_acquire_lock( kmp_lock_t *lck, kmp_int32 gtid )
587 return __kmp_acquire_ticket_lock( lck, gtid );
591 __kmp_test_lock( kmp_lock_t *lck, kmp_int32 gtid )
593 return __kmp_test_ticket_lock( lck, gtid );
597 __kmp_release_lock( kmp_lock_t *lck, kmp_int32 gtid )
599 __kmp_release_ticket_lock( lck, gtid );
603 __kmp_init_lock( kmp_lock_t *lck )
605 __kmp_init_ticket_lock( lck );
609 __kmp_destroy_lock( kmp_lock_t *lck )
611 __kmp_destroy_ticket_lock( lck );
633 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX 640 #if KMP_USE_ADAPTIVE_LOCKS 642 #endif // KMP_USE_ADAPTIVE_LOCKS 645 typedef enum kmp_lock_kind kmp_lock_kind_t;
647 extern kmp_lock_kind_t __kmp_user_lock_kind;
649 union kmp_user_lock {
652 kmp_futex_lock_t futex;
654 kmp_ticket_lock_t ticket;
655 kmp_queuing_lock_t queuing;
656 kmp_drdpa_lock_t drdpa;
657 #if KMP_USE_ADAPTIVE_LOCKS 658 kmp_adaptive_lock_t adaptive;
659 #endif // KMP_USE_ADAPTIVE_LOCKS 660 kmp_lock_pool_t pool;
663 typedef union kmp_user_lock *kmp_user_lock_p;
665 #if ! KMP_USE_DYNAMIC_LOCK 667 extern size_t __kmp_base_user_lock_size;
668 extern size_t __kmp_user_lock_size;
670 extern kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck );
672 static inline kmp_int32
673 __kmp_get_user_lock_owner( kmp_user_lock_p lck )
675 KMP_DEBUG_ASSERT( __kmp_get_user_lock_owner_ != NULL );
676 return ( *__kmp_get_user_lock_owner_ )( lck );
679 extern int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
681 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 683 #define __kmp_acquire_user_lock_with_checks(lck,gtid) \ 684 if (__kmp_user_lock_kind == lk_tas) { \ 685 if ( __kmp_env_consistency_check ) { \ 686 char const * const func = "omp_set_lock"; \ 687 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE ) \ 688 && lck->tas.lk.depth_locked != -1 ) { \ 689 KMP_FATAL( LockNestableUsedAsSimple, func ); \ 691 if ( ( gtid >= 0 ) && ( lck->tas.lk.poll - 1 == gtid ) ) { \ 692 KMP_FATAL( LockIsAlreadyOwned, func ); \ 695 if ( ( lck->tas.lk.poll != 0 ) || \ 696 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 698 KMP_FSYNC_PREPARE( lck ); \ 699 KMP_INIT_YIELD( spins ); \ 700 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 703 KMP_YIELD_SPIN( spins ); \ 705 while ( ( lck->tas.lk.poll != 0 ) || \ 706 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 707 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 710 KMP_YIELD_SPIN( spins ); \ 714 KMP_FSYNC_ACQUIRED( lck ); \ 716 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL ); \ 717 ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid ); \ 722 __kmp_acquire_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
724 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL );
725 return ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid );
729 extern int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
731 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 733 #include "kmp_i18n.h" 734 extern int __kmp_env_consistency_check;
736 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
738 if ( __kmp_user_lock_kind == lk_tas ) {
739 if ( __kmp_env_consistency_check ) {
740 char const *
const func =
"omp_test_lock";
741 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
742 && lck->tas.lk.depth_locked != -1 ) {
743 KMP_FATAL( LockNestableUsedAsSimple, func );
746 return ( ( lck->tas.lk.poll == 0 ) &&
747 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
749 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
750 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
755 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
757 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
758 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
762 extern int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
765 __kmp_release_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
767 KMP_DEBUG_ASSERT( __kmp_release_user_lock_with_checks_ != NULL );
768 ( *__kmp_release_user_lock_with_checks_ ) ( lck, gtid );
771 extern void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck );
774 __kmp_init_user_lock_with_checks( kmp_user_lock_p lck )
776 KMP_DEBUG_ASSERT( __kmp_init_user_lock_with_checks_ != NULL );
777 ( *__kmp_init_user_lock_with_checks_ )( lck );
784 extern void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck );
787 __kmp_destroy_user_lock( kmp_user_lock_p lck )
789 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_ != NULL );
790 ( *__kmp_destroy_user_lock_ )( lck );
793 extern void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck );
796 __kmp_destroy_user_lock_with_checks( kmp_user_lock_p lck )
798 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_with_checks_ != NULL );
799 ( *__kmp_destroy_user_lock_with_checks_ )( lck );
802 extern int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
804 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 806 #define __kmp_acquire_nested_user_lock_with_checks(lck,gtid,depth) \ 807 if (__kmp_user_lock_kind == lk_tas) { \ 808 if ( __kmp_env_consistency_check ) { \ 809 char const * const func = "omp_set_nest_lock"; \ 810 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE ) \ 811 && lck->tas.lk.depth_locked == -1 ) { \ 812 KMP_FATAL( LockSimpleUsedAsNestable, func ); \ 815 if ( lck->tas.lk.poll - 1 == gtid ) { \ 816 lck->tas.lk.depth_locked += 1; \ 817 *depth = KMP_LOCK_ACQUIRED_NEXT; \ 819 if ( ( lck->tas.lk.poll != 0 ) || \ 820 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 822 KMP_FSYNC_PREPARE( lck ); \ 823 KMP_INIT_YIELD( spins ); \ 824 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 827 KMP_YIELD_SPIN( spins ); \ 829 while ( ( lck->tas.lk.poll != 0 ) || \ 830 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 831 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 834 KMP_YIELD_SPIN( spins ); \ 838 lck->tas.lk.depth_locked = 1; \ 839 *depth = KMP_LOCK_ACQUIRED_FIRST; \ 841 KMP_FSYNC_ACQUIRED( lck ); \ 843 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL ); \ 844 *depth = ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid ); \ 849 __kmp_acquire_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid,
int* depth )
851 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL );
852 *depth = ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid );
856 extern int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
858 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 860 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
862 if ( __kmp_user_lock_kind == lk_tas ) {
864 if ( __kmp_env_consistency_check ) {
865 char const *
const func =
"omp_test_nest_lock";
866 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE )
867 && lck->tas.lk.depth_locked == -1 ) {
868 KMP_FATAL( LockSimpleUsedAsNestable, func );
871 KMP_DEBUG_ASSERT( gtid >= 0 );
872 if ( lck->tas.lk.poll - 1 == gtid ) {
873 return ++lck->tas.lk.depth_locked;
875 retval = ( ( lck->tas.lk.poll == 0 ) &&
876 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
879 lck->tas.lk.depth_locked = 1;
883 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
884 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
889 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
891 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
892 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
896 extern int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
899 __kmp_release_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
901 KMP_DEBUG_ASSERT( __kmp_release_nested_user_lock_with_checks_ != NULL );
902 return ( *__kmp_release_nested_user_lock_with_checks_ )( lck, gtid );
905 extern void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
907 static inline void __kmp_init_nested_user_lock_with_checks( kmp_user_lock_p lck )
909 KMP_DEBUG_ASSERT( __kmp_init_nested_user_lock_with_checks_ != NULL );
910 ( *__kmp_init_nested_user_lock_with_checks_ )( lck );
913 extern void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
916 __kmp_destroy_nested_user_lock_with_checks( kmp_user_lock_p lck )
918 KMP_DEBUG_ASSERT( __kmp_destroy_nested_user_lock_with_checks_ != NULL );
919 ( *__kmp_destroy_nested_user_lock_with_checks_ )( lck );
937 extern int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck );
941 extern const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck );
944 __kmp_get_user_lock_location( kmp_user_lock_p lck )
946 if ( __kmp_get_user_lock_location_ != NULL ) {
947 return ( *__kmp_get_user_lock_location_ )( lck );
954 extern void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc );
957 __kmp_set_user_lock_location( kmp_user_lock_p lck,
const ident_t *loc )
959 if ( __kmp_set_user_lock_location_ != NULL ) {
960 ( *__kmp_set_user_lock_location_ )( lck, loc );
964 extern kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck );
966 extern void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags );
969 __kmp_set_user_lock_flags( kmp_user_lock_p lck, kmp_lock_flags_t flags )
971 if ( __kmp_set_user_lock_flags_ != NULL ) {
972 ( *__kmp_set_user_lock_flags_ )( lck, flags );
979 extern void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind );
984 #define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) { \ 985 __kmp_acquire##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 986 __kmp_acquire##nest##kind##_##suffix; \ 987 __kmp_release##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 988 __kmp_release##nest##kind##_##suffix; \ 989 __kmp_test##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 990 __kmp_test##nest##kind##_##suffix; \ 991 __kmp_init##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \ 992 __kmp_init##nest##kind##_##suffix; \ 993 __kmp_destroy##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \ 994 __kmp_destroy##nest##kind##_##suffix; \ 997 #define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock) 998 #define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks) 999 #define KMP_BIND_NESTED_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock) 1000 #define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks) 1028 struct kmp_lock_table {
1029 kmp_lock_index_t used;
1030 kmp_lock_index_t allocated;
1031 kmp_user_lock_p * table;
1034 typedef struct kmp_lock_table kmp_lock_table_t;
1036 extern kmp_lock_table_t __kmp_user_lock_table;
1037 extern kmp_user_lock_p __kmp_lock_pool;
1039 struct kmp_block_of_locks {
1040 struct kmp_block_of_locks * next_block;
1044 typedef struct kmp_block_of_locks kmp_block_of_locks_t;
1046 extern kmp_block_of_locks_t *__kmp_lock_blocks;
1047 extern int __kmp_num_locks_in_block;
1049 extern kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid, kmp_lock_flags_t flags );
1050 extern void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck );
1051 extern kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func );
1052 extern void __kmp_cleanup_user_locks();
1054 #define KMP_CHECK_USER_LOCK_INIT() \ 1056 if ( ! TCR_4( __kmp_init_user_locks ) ) { \ 1057 __kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); \ 1058 if ( ! TCR_4( __kmp_init_user_locks ) ) { \ 1059 TCW_4( __kmp_init_user_locks, TRUE ); \ 1061 __kmp_release_bootstrap_lock( &__kmp_initz_lock ); \ 1065 #endif // KMP_USE_DYNAMIC_LOCK 1070 #if KMP_USE_DYNAMIC_LOCK 1106 #define KMP_USE_INLINED_TAS (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1 1107 #define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0 1114 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) 1115 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \ 1116 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1117 m(nested_queuing, a) m(nested_drdpa, a) 1119 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) 1120 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \ 1121 m(nested_tas, a) m(nested_ticket, a) \ 1122 m(nested_queuing, a) m(nested_drdpa, a) 1123 # endif // KMP_USE_FUTEX 1124 # define KMP_LAST_D_LOCK lockseq_hle 1127 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) 1128 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \ 1129 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1130 m(nested_queuing, a) m(nested_drdpa, a) 1131 # define KMP_LAST_D_LOCK lockseq_futex 1133 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) 1134 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \ 1135 m(nested_tas, a) m(nested_ticket, a) \ 1136 m(nested_queuing, a) m(nested_drdpa, a) 1137 # define KMP_LAST_D_LOCK lockseq_tas 1138 # endif // KMP_USE_FUTEX 1139 #endif // KMP_USE_TSX 1142 #define KMP_LOCK_SHIFT 8 // number of low bits to be used as tag for direct locks 1143 #define KMP_FIRST_D_LOCK lockseq_tas 1144 #define KMP_FIRST_I_LOCK lockseq_ticket 1145 #define KMP_LAST_I_LOCK lockseq_nested_drdpa 1146 #define KMP_NUM_I_LOCKS (locktag_nested_drdpa+1) // number of indirect lock types 1149 typedef kmp_uint32 kmp_dyna_lock_t;
1154 lockseq_indirect = 0,
1155 #define expand_seq(l,a) lockseq_##l, 1156 KMP_FOREACH_D_LOCK(expand_seq, 0)
1157 KMP_FOREACH_I_LOCK(expand_seq, 0)
1159 } kmp_dyna_lockseq_t;
1163 #define expand_tag(l,a) locktag_##l, 1164 KMP_FOREACH_I_LOCK(expand_tag, 0)
1166 } kmp_indirect_locktag_t;
1169 #define KMP_IS_D_LOCK(seq) ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK) 1170 #define KMP_IS_I_LOCK(seq) ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK) 1171 #define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq) - KMP_FIRST_I_LOCK) 1172 #define KMP_GET_D_TAG(seq) ((seq)<<1 | 1) 1176 #define expand_tag(l,a) locktag_##l = KMP_GET_D_TAG(lockseq_##l), 1177 KMP_FOREACH_D_LOCK(expand_tag, 0)
1179 } kmp_direct_locktag_t;
1183 kmp_user_lock_p lock;
1184 kmp_indirect_locktag_t type;
1185 } kmp_indirect_lock_t;
1188 extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
1189 extern void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *);
1190 extern void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32);
1191 extern int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32);
1192 extern int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32);
1195 extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
1196 extern void (*__kmp_indirect_destroy[])(kmp_user_lock_p);
1197 extern void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32);
1198 extern int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32);
1199 extern int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32);
1202 #define KMP_EXTRACT_D_TAG(l) (*((kmp_dyna_lock_t *)(l)) & ((1<<KMP_LOCK_SHIFT)-1) & -(*((kmp_dyna_lock_t *)(l)) & 1)) 1205 #define KMP_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1) 1208 #define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)] 1211 #define KMP_I_LOCK_FUNC(l, op) __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type] 1214 #define KMP_INIT_D_LOCK(l, seq) __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq) 1217 #define KMP_INIT_I_LOCK(l, seq) __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq) 1220 #define KMP_LOCK_FREE(type) (locktag_##type) 1223 #define KMP_LOCK_BUSY(v, type) ((v)<<KMP_LOCK_SHIFT | locktag_##type) 1226 #define KMP_LOCK_STRIP(v) ((v)>>KMP_LOCK_SHIFT) 1229 extern void __kmp_init_dynamic_user_locks();
1232 extern kmp_indirect_lock_t * __kmp_allocate_indirect_lock(
void **, kmp_int32, kmp_indirect_locktag_t);
1235 extern void __kmp_cleanup_indirect_user_locks();
1238 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
1241 extern void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *);
1242 #define KMP_SET_I_LOCK_LOCATION(lck, loc) { \ 1243 if (__kmp_indirect_set_location[(lck)->type] != NULL) \ 1244 __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \ 1248 extern void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t);
1249 #define KMP_SET_I_LOCK_FLAGS(lck, flag) { \ 1250 if (__kmp_indirect_set_flags[(lck)->type] != NULL) \ 1251 __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \ 1255 extern const ident_t * (*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p);
1256 #define KMP_GET_I_LOCK_LOCATION(lck) ( __kmp_indirect_get_location[(lck)->type] != NULL \ 1257 ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \ 1261 extern kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p);
1262 #define KMP_GET_I_LOCK_FLAGS(lck) ( __kmp_indirect_get_flags[(lck)->type] != NULL \ 1263 ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \ 1266 #define KMP_I_LOCK_CHUNK 1024 // number of kmp_indirect_lock_t objects to be allocated together 1269 typedef struct kmp_indirect_lock_table {
1270 kmp_indirect_lock_t **table;
1271 kmp_lock_index_t size;
1272 kmp_lock_index_t next;
1273 } kmp_indirect_lock_table_t;
1275 extern kmp_indirect_lock_table_t __kmp_i_lock_table;
1278 #define KMP_GET_I_LOCK(index) (*(__kmp_i_lock_table.table + (index)/KMP_I_LOCK_CHUNK) + (index)%KMP_I_LOCK_CHUNK) 1283 extern int __kmp_num_locks_in_block;
1286 #define KMP_LOOKUP_I_LOCK(l) ( (OMP_LOCK_T_SIZE < sizeof(void *)) \ 1287 ? KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(l)) \ 1288 : *((kmp_indirect_lock_t **)(l)) ) 1292 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
1294 #else // KMP_USE_DYNAMIC_LOCK 1296 # define KMP_LOCK_BUSY(v, type) (v) 1297 # define KMP_LOCK_FREE(type) 0 1298 # define KMP_LOCK_STRIP(v) (v) 1300 #endif // KMP_USE_DYNAMIC_LOCK 1305 kmp_uint32 max_backoff;
1306 kmp_uint32 min_tick;
1310 extern kmp_backoff_t __kmp_spin_backoff_params;
1313 extern void __kmp_spin_backoff(kmp_backoff_t *);
1317 #endif // __cplusplus