23 #include "kmp_debug.h" 33 #define KMP_PAD(type, sz) (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1)) 34 #define KMP_GTID_DNE (-2) 55 #if KMP_OS_LINUX && defined(KMP_GOMP_COMPAT) 56 # define OMP_LOCK_T_SIZE sizeof(int) 57 # define OMP_NEST_LOCK_T_SIZE sizeof(void *) 59 # define OMP_LOCK_T_SIZE sizeof(void *) 60 # define OMP_NEST_LOCK_T_SIZE sizeof(void *) 68 #define OMP_CRITICAL_SIZE sizeof(void *) 69 #define INTEL_CRITICAL_SIZE 32 74 typedef kmp_uint32 kmp_lock_flags_t;
76 #define kmp_lf_critical_section 1 81 typedef kmp_uint32 kmp_lock_index_t;
87 struct kmp_lock_pool {
88 union kmp_user_lock *next;
89 kmp_lock_index_t index;
92 typedef struct kmp_lock_pool kmp_lock_pool_t;
95 extern void __kmp_validate_locks(
void );
139 struct kmp_base_tas_lock {
140 volatile kmp_int32 poll;
142 kmp_int32 depth_locked;
145 typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
148 kmp_base_tas_lock_t lk;
149 kmp_lock_pool_t pool;
154 typedef union kmp_tas_lock kmp_tas_lock_t;
160 #define KMP_TAS_LOCK_INITIALIZER( lock ) { { 0, 0 } } 162 extern int __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
163 extern int __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
164 extern int __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
165 extern void __kmp_init_tas_lock( kmp_tas_lock_t *lck );
166 extern void __kmp_destroy_tas_lock( kmp_tas_lock_t *lck );
168 extern int __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
169 extern int __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
170 extern int __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
171 extern void __kmp_init_nested_tas_lock( kmp_tas_lock_t *lck );
172 extern void __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck );
174 #define KMP_LOCK_RELEASED 1 175 #define KMP_LOCK_STILL_HELD 0 176 #define KMP_LOCK_ACQUIRED_FIRST 1 177 #define KMP_LOCK_ACQUIRED_NEXT 0 179 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 193 struct kmp_base_futex_lock {
194 volatile kmp_int32 poll;
197 kmp_int32 depth_locked;
200 typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
202 union kmp_futex_lock {
203 kmp_base_futex_lock_t lk;
204 kmp_lock_pool_t pool;
209 typedef union kmp_futex_lock kmp_futex_lock_t;
215 #define KMP_FUTEX_LOCK_INITIALIZER( lock ) { { 0, 0 } } 217 extern int __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
218 extern int __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
219 extern int __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
220 extern void __kmp_init_futex_lock( kmp_futex_lock_t *lck );
221 extern void __kmp_destroy_futex_lock( kmp_futex_lock_t *lck );
223 extern int __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
224 extern int __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
225 extern int __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
226 extern void __kmp_init_nested_futex_lock( kmp_futex_lock_t *lck );
227 extern void __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck );
229 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 236 struct kmp_base_ticket_lock {
238 volatile union kmp_ticket_lock * initialized;
240 volatile kmp_uint32 next_ticket;
241 volatile kmp_uint32 now_serving;
242 volatile kmp_int32 owner_id;
243 kmp_int32 depth_locked;
244 kmp_lock_flags_t flags;
247 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
249 union KMP_ALIGN_CACHE kmp_ticket_lock {
250 kmp_base_ticket_lock_t lk;
251 kmp_lock_pool_t pool;
253 char lk_pad[ KMP_PAD( kmp_base_ticket_lock_t, CACHE_LINE ) ];
256 typedef union kmp_ticket_lock kmp_ticket_lock_t;
263 #define KMP_TICKET_LOCK_INITIALIZER( lock ) { { (kmp_ticket_lock_t *) & (lock), NULL, 0, 0, 0, -1 } } 265 extern int __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
266 extern int __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
267 extern int __kmp_test_ticket_lock_with_cheks( kmp_ticket_lock_t *lck, kmp_int32 gtid );
268 extern int __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
269 extern void __kmp_init_ticket_lock( kmp_ticket_lock_t *lck );
270 extern void __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck );
272 extern int __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
273 extern int __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
274 extern int __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
275 extern void __kmp_init_nested_ticket_lock( kmp_ticket_lock_t *lck );
276 extern void __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck );
283 #if KMP_USE_ADAPTIVE_LOCKS 285 struct kmp_adaptive_lock_info;
287 typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
289 #if KMP_DEBUG_ADAPTIVE_LOCKS 291 struct kmp_adaptive_lock_statistics {
293 kmp_adaptive_lock_info_t * next;
294 kmp_adaptive_lock_info_t * prev;
297 kmp_uint32 successfulSpeculations;
298 kmp_uint32 hardFailedSpeculations;
299 kmp_uint32 softFailedSpeculations;
300 kmp_uint32 nonSpeculativeAcquires;
301 kmp_uint32 nonSpeculativeAcquireAttempts;
302 kmp_uint32 lemmingYields;
305 typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
307 extern void __kmp_print_speculative_stats();
308 extern void __kmp_init_speculative_stats();
310 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 312 struct kmp_adaptive_lock_info
319 kmp_uint32
volatile badness;
320 kmp_uint32
volatile acquire_attempts;
322 kmp_uint32 max_badness;
323 kmp_uint32 max_soft_retries;
325 #if KMP_DEBUG_ADAPTIVE_LOCKS 326 kmp_adaptive_lock_statistics_t
volatile stats;
330 #endif // KMP_USE_ADAPTIVE_LOCKS 333 struct kmp_base_queuing_lock {
336 volatile union kmp_queuing_lock *initialized;
342 volatile kmp_int32 tail_id;
344 volatile kmp_int32 head_id;
347 volatile kmp_uint32 next_ticket;
348 volatile kmp_uint32 now_serving;
349 volatile kmp_int32 owner_id;
350 kmp_int32 depth_locked;
352 kmp_lock_flags_t flags;
355 typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
357 KMP_BUILD_ASSERT( offsetof( kmp_base_queuing_lock_t, tail_id ) % 8 == 0 );
359 union KMP_ALIGN_CACHE kmp_queuing_lock {
360 kmp_base_queuing_lock_t lk;
361 kmp_lock_pool_t pool;
363 char lk_pad[ KMP_PAD( kmp_base_queuing_lock_t, CACHE_LINE ) ];
366 typedef union kmp_queuing_lock kmp_queuing_lock_t;
368 extern int __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
369 extern int __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
370 extern int __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
371 extern void __kmp_init_queuing_lock( kmp_queuing_lock_t *lck );
372 extern void __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck );
374 extern int __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
375 extern int __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
376 extern int __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
377 extern void __kmp_init_nested_queuing_lock( kmp_queuing_lock_t *lck );
378 extern void __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck );
380 #if KMP_USE_ADAPTIVE_LOCKS 385 struct kmp_base_adaptive_lock {
386 kmp_base_queuing_lock qlk;
387 KMP_ALIGN(CACHE_LINE)
388 kmp_adaptive_lock_info_t adaptive;
391 typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
393 union KMP_ALIGN_CACHE kmp_adaptive_lock {
394 kmp_base_adaptive_lock_t lk;
395 kmp_lock_pool_t pool;
397 char lk_pad[ KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE) ];
399 typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
401 # define GET_QLK_PTR(l) ((kmp_queuing_lock_t *) & (l)->lk.qlk) 403 #endif // KMP_USE_ADAPTIVE_LOCKS 409 struct kmp_base_drdpa_lock {
420 volatile union kmp_drdpa_lock * initialized;
422 volatile struct kmp_lock_poll {
425 volatile kmp_uint64 mask;
426 kmp_uint64 cleanup_ticket;
427 volatile struct kmp_lock_poll * old_polls;
428 kmp_uint32 num_polls;
436 volatile kmp_uint64 next_ticket;
455 kmp_uint64 now_serving;
456 volatile kmp_uint32 owner_id;
457 kmp_int32 depth_locked;
458 kmp_lock_flags_t flags;
461 typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
463 union KMP_ALIGN_CACHE kmp_drdpa_lock {
464 kmp_base_drdpa_lock_t lk;
465 kmp_lock_pool_t pool;
467 char lk_pad[ KMP_PAD( kmp_base_drdpa_lock_t, CACHE_LINE ) ];
470 typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
472 extern int __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
473 extern int __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
474 extern int __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
475 extern void __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck );
476 extern void __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck );
478 extern int __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
479 extern int __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
480 extern int __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
481 extern void __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
482 extern void __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
500 typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
502 #define KMP_BOOTSTRAP_LOCK_INITIALIZER( lock ) KMP_TICKET_LOCK_INITIALIZER( (lock) ) 505 __kmp_acquire_bootstrap_lock( kmp_bootstrap_lock_t *lck )
507 return __kmp_acquire_ticket_lock( lck, KMP_GTID_DNE );
511 __kmp_test_bootstrap_lock( kmp_bootstrap_lock_t *lck )
513 return __kmp_test_ticket_lock( lck, KMP_GTID_DNE );
517 __kmp_release_bootstrap_lock( kmp_bootstrap_lock_t *lck )
519 __kmp_release_ticket_lock( lck, KMP_GTID_DNE );
523 __kmp_init_bootstrap_lock( kmp_bootstrap_lock_t *lck )
525 __kmp_init_ticket_lock( lck );
529 __kmp_destroy_bootstrap_lock( kmp_bootstrap_lock_t *lck )
531 __kmp_destroy_ticket_lock( lck );
547 typedef kmp_ticket_lock_t kmp_lock_t;
550 __kmp_acquire_lock( kmp_lock_t *lck, kmp_int32 gtid )
552 return __kmp_acquire_ticket_lock( lck, gtid );
556 __kmp_test_lock( kmp_lock_t *lck, kmp_int32 gtid )
558 return __kmp_test_ticket_lock( lck, gtid );
562 __kmp_release_lock( kmp_lock_t *lck, kmp_int32 gtid )
564 __kmp_release_ticket_lock( lck, gtid );
568 __kmp_init_lock( kmp_lock_t *lck )
570 __kmp_init_ticket_lock( lck );
574 __kmp_destroy_lock( kmp_lock_t *lck )
576 __kmp_destroy_ticket_lock( lck );
595 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 598 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX 605 #if KMP_USE_ADAPTIVE_LOCKS 607 #endif // KMP_USE_ADAPTIVE_LOCKS 610 typedef enum kmp_lock_kind kmp_lock_kind_t;
612 extern kmp_lock_kind_t __kmp_user_lock_kind;
614 union kmp_user_lock {
616 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 617 kmp_futex_lock_t futex;
619 kmp_ticket_lock_t ticket;
620 kmp_queuing_lock_t queuing;
621 kmp_drdpa_lock_t drdpa;
622 #if KMP_USE_ADAPTIVE_LOCKS 623 kmp_adaptive_lock_t adaptive;
624 #endif // KMP_USE_ADAPTIVE_LOCKS 625 kmp_lock_pool_t pool;
628 typedef union kmp_user_lock *kmp_user_lock_p;
630 #if ! KMP_USE_DYNAMIC_LOCK 632 extern size_t __kmp_base_user_lock_size;
633 extern size_t __kmp_user_lock_size;
635 extern kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck );
637 static inline kmp_int32
638 __kmp_get_user_lock_owner( kmp_user_lock_p lck )
640 KMP_DEBUG_ASSERT( __kmp_get_user_lock_owner_ != NULL );
641 return ( *__kmp_get_user_lock_owner_ )( lck );
644 extern int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
646 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 648 #define __kmp_acquire_user_lock_with_checks(lck,gtid) \ 649 if (__kmp_user_lock_kind == lk_tas) { \ 650 if ( __kmp_env_consistency_check ) { \ 651 char const * const func = "omp_set_lock"; \ 652 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE ) \ 653 && lck->tas.lk.depth_locked != -1 ) { \ 654 KMP_FATAL( LockNestableUsedAsSimple, func ); \ 656 if ( ( gtid >= 0 ) && ( lck->tas.lk.poll - 1 == gtid ) ) { \ 657 KMP_FATAL( LockIsAlreadyOwned, func ); \ 660 if ( ( lck->tas.lk.poll != 0 ) || \ 661 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 663 KMP_FSYNC_PREPARE( lck ); \ 664 KMP_INIT_YIELD( spins ); \ 665 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 668 KMP_YIELD_SPIN( spins ); \ 670 while ( ( lck->tas.lk.poll != 0 ) || \ 671 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 672 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 675 KMP_YIELD_SPIN( spins ); \ 679 KMP_FSYNC_ACQUIRED( lck ); \ 681 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL ); \ 682 ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid ); \ 687 __kmp_acquire_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
689 KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL );
690 return ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid );
694 extern int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
696 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 698 #include "kmp_i18n.h" 699 extern int __kmp_env_consistency_check;
701 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
703 if ( __kmp_user_lock_kind == lk_tas ) {
704 if ( __kmp_env_consistency_check ) {
705 char const *
const func =
"omp_test_lock";
706 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
707 && lck->tas.lk.depth_locked != -1 ) {
708 KMP_FATAL( LockNestableUsedAsSimple, func );
711 return ( ( lck->tas.lk.poll == 0 ) &&
712 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
714 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
715 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
720 __kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
722 KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
723 return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
727 extern int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
730 __kmp_release_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
732 KMP_DEBUG_ASSERT( __kmp_release_user_lock_with_checks_ != NULL );
733 ( *__kmp_release_user_lock_with_checks_ ) ( lck, gtid );
736 extern void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck );
739 __kmp_init_user_lock_with_checks( kmp_user_lock_p lck )
741 KMP_DEBUG_ASSERT( __kmp_init_user_lock_with_checks_ != NULL );
742 ( *__kmp_init_user_lock_with_checks_ )( lck );
749 extern void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck );
752 __kmp_destroy_user_lock( kmp_user_lock_p lck )
754 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_ != NULL );
755 ( *__kmp_destroy_user_lock_ )( lck );
758 extern void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck );
761 __kmp_destroy_user_lock_with_checks( kmp_user_lock_p lck )
763 KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_with_checks_ != NULL );
764 ( *__kmp_destroy_user_lock_with_checks_ )( lck );
767 extern int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
769 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 771 #define __kmp_acquire_nested_user_lock_with_checks(lck,gtid,depth) \ 772 if (__kmp_user_lock_kind == lk_tas) { \ 773 if ( __kmp_env_consistency_check ) { \ 774 char const * const func = "omp_set_nest_lock"; \ 775 if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE ) \ 776 && lck->tas.lk.depth_locked == -1 ) { \ 777 KMP_FATAL( LockSimpleUsedAsNestable, func ); \ 780 if ( lck->tas.lk.poll - 1 == gtid ) { \ 781 lck->tas.lk.depth_locked += 1; \ 782 *depth = KMP_LOCK_ACQUIRED_NEXT; \ 784 if ( ( lck->tas.lk.poll != 0 ) || \ 785 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 787 KMP_FSYNC_PREPARE( lck ); \ 788 KMP_INIT_YIELD( spins ); \ 789 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 792 KMP_YIELD_SPIN( spins ); \ 794 while ( ( lck->tas.lk.poll != 0 ) || \ 795 ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \ 796 if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \ 799 KMP_YIELD_SPIN( spins ); \ 803 lck->tas.lk.depth_locked = 1; \ 804 *depth = KMP_LOCK_ACQUIRED_FIRST; \ 806 KMP_FSYNC_ACQUIRED( lck ); \ 808 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL ); \ 809 *depth = ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid ); \ 814 __kmp_acquire_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid,
int* depth )
816 KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL );
817 *depth = ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid );
821 extern int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
823 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 825 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
827 if ( __kmp_user_lock_kind == lk_tas ) {
829 if ( __kmp_env_consistency_check ) {
830 char const *
const func =
"omp_test_nest_lock";
831 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE )
832 && lck->tas.lk.depth_locked == -1 ) {
833 KMP_FATAL( LockSimpleUsedAsNestable, func );
836 KMP_DEBUG_ASSERT( gtid >= 0 );
837 if ( lck->tas.lk.poll - 1 == gtid ) {
838 return ++lck->tas.lk.depth_locked;
840 retval = ( ( lck->tas.lk.poll == 0 ) &&
841 KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
844 lck->tas.lk.depth_locked = 1;
848 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
849 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
854 __kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
856 KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
857 return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
861 extern int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
864 __kmp_release_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
866 KMP_DEBUG_ASSERT( __kmp_release_nested_user_lock_with_checks_ != NULL );
867 return ( *__kmp_release_nested_user_lock_with_checks_ )( lck, gtid );
870 extern void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
872 static inline void __kmp_init_nested_user_lock_with_checks( kmp_user_lock_p lck )
874 KMP_DEBUG_ASSERT( __kmp_init_nested_user_lock_with_checks_ != NULL );
875 ( *__kmp_init_nested_user_lock_with_checks_ )( lck );
878 extern void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
881 __kmp_destroy_nested_user_lock_with_checks( kmp_user_lock_p lck )
883 KMP_DEBUG_ASSERT( __kmp_destroy_nested_user_lock_with_checks_ != NULL );
884 ( *__kmp_destroy_nested_user_lock_with_checks_ )( lck );
902 extern int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck );
906 extern const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck );
909 __kmp_get_user_lock_location( kmp_user_lock_p lck )
911 if ( __kmp_get_user_lock_location_ != NULL ) {
912 return ( *__kmp_get_user_lock_location_ )( lck );
919 extern void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc );
922 __kmp_set_user_lock_location( kmp_user_lock_p lck,
const ident_t *loc )
924 if ( __kmp_set_user_lock_location_ != NULL ) {
925 ( *__kmp_set_user_lock_location_ )( lck, loc );
929 extern kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck );
931 extern void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags );
934 __kmp_set_user_lock_flags( kmp_user_lock_p lck, kmp_lock_flags_t flags )
936 if ( __kmp_set_user_lock_flags_ != NULL ) {
937 ( *__kmp_set_user_lock_flags_ )( lck, flags );
944 extern void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind );
949 #define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) { \ 950 __kmp_acquire##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 951 __kmp_acquire##nest##kind##_##suffix; \ 952 __kmp_release##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 953 __kmp_release##nest##kind##_##suffix; \ 954 __kmp_test##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \ 955 __kmp_test##nest##kind##_##suffix; \ 956 __kmp_init##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \ 957 __kmp_init##nest##kind##_##suffix; \ 958 __kmp_destroy##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \ 959 __kmp_destroy##nest##kind##_##suffix; \ 962 #define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock) 963 #define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks) 964 #define KMP_BIND_NESTED_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock) 965 #define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks) 993 struct kmp_lock_table {
994 kmp_lock_index_t used;
995 kmp_lock_index_t allocated;
996 kmp_user_lock_p * table;
999 typedef struct kmp_lock_table kmp_lock_table_t;
1001 extern kmp_lock_table_t __kmp_user_lock_table;
1002 extern kmp_user_lock_p __kmp_lock_pool;
1004 struct kmp_block_of_locks {
1005 struct kmp_block_of_locks * next_block;
1009 typedef struct kmp_block_of_locks kmp_block_of_locks_t;
1011 extern kmp_block_of_locks_t *__kmp_lock_blocks;
1012 extern int __kmp_num_locks_in_block;
1014 extern kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid, kmp_lock_flags_t flags );
1015 extern void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck );
1016 extern kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func );
1017 extern void __kmp_cleanup_user_locks();
1019 #define KMP_CHECK_USER_LOCK_INIT() \ 1021 if ( ! TCR_4( __kmp_init_user_locks ) ) { \ 1022 __kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); \ 1023 if ( ! TCR_4( __kmp_init_user_locks ) ) { \ 1024 TCW_4( __kmp_init_user_locks, TRUE ); \ 1026 __kmp_release_bootstrap_lock( &__kmp_initz_lock ); \ 1030 #endif // KMP_USE_DYNAMIC_LOCK 1035 #if KMP_USE_DYNAMIC_LOCK 1071 #define KMP_USE_FUTEX (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)) 1072 #define KMP_USE_INLINED_TAS (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1 1073 #define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0 1080 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) 1081 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \ 1082 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1083 m(nested_queuing, a) m(nested_drdpa, a) 1085 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) 1086 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \ 1087 m(nested_tas, a) m(nested_ticket, a) \ 1088 m(nested_queuing, a) m(nested_drdpa, a) 1089 # endif // KMP_USE_FUTEX 1090 # define KMP_LAST_D_LOCK lockseq_hle 1093 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) 1094 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \ 1095 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1096 m(nested_queuing, a) m(nested_drdpa, a) 1097 # define KMP_LAST_D_LOCK lockseq_futex 1099 # define KMP_FOREACH_D_LOCK(m, a) m(tas, a) 1100 # define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \ 1101 m(nested_tas, a) m(nested_ticket, a) \ 1102 m(nested_queuing, a) m(nested_drdpa, a) 1103 # define KMP_LAST_D_LOCK lockseq_tas 1104 # endif // KMP_USE_FUTEX 1105 #endif // KMP_USE_TSX 1108 #define KMP_LOCK_SHIFT 8 // number of low bits to be used as tag for direct locks 1109 #define KMP_FIRST_D_LOCK lockseq_tas 1110 #define KMP_FIRST_I_LOCK lockseq_ticket 1111 #define KMP_LAST_I_LOCK lockseq_nested_drdpa 1112 #define KMP_NUM_I_LOCKS (locktag_nested_drdpa+1) // number of indirect lock types 1115 typedef kmp_uint32 kmp_dyna_lock_t;
1120 lockseq_indirect = 0,
1121 #define expand_seq(l,a) lockseq_##l, 1122 KMP_FOREACH_D_LOCK(expand_seq, 0)
1123 KMP_FOREACH_I_LOCK(expand_seq, 0)
1125 } kmp_dyna_lockseq_t;
1129 #define expand_tag(l,a) locktag_##l, 1130 KMP_FOREACH_I_LOCK(expand_tag, 0)
1132 } kmp_indirect_locktag_t;
1135 #define KMP_IS_D_LOCK(seq) ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK) 1136 #define KMP_IS_I_LOCK(seq) ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK) 1137 #define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq) - KMP_FIRST_I_LOCK) 1138 #define KMP_GET_D_TAG(seq) ((seq)<<1 | 1) 1142 #define expand_tag(l,a) locktag_##l = KMP_GET_D_TAG(lockseq_##l), 1143 KMP_FOREACH_D_LOCK(expand_tag, 0)
1145 } kmp_direct_locktag_t;
1149 kmp_user_lock_p lock;
1150 kmp_indirect_locktag_t type;
1151 } kmp_indirect_lock_t;
1154 extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
1155 extern void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *);
1156 extern void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32);
1157 extern int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32);
1158 extern int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32);
1161 extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
1162 extern void (*__kmp_indirect_destroy[])(kmp_user_lock_p);
1163 extern void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32);
1164 extern int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32);
1165 extern int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32);
1168 #define KMP_EXTRACT_D_TAG(l) (*((kmp_dyna_lock_t *)(l)) & ((1<<KMP_LOCK_SHIFT)-1) & -(*((kmp_dyna_lock_t *)(l)) & 1)) 1171 #define KMP_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1) 1174 #define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)] 1177 #define KMP_I_LOCK_FUNC(l, op) __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type] 1180 #define KMP_INIT_D_LOCK(l, seq) __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq) 1183 #define KMP_INIT_I_LOCK(l, seq) __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq) 1186 #define KMP_LOCK_FREE(type) (locktag_##type) 1189 #define KMP_LOCK_BUSY(v, type) ((v)<<KMP_LOCK_SHIFT | locktag_##type) 1192 #define KMP_LOCK_STRIP(v) ((v)>>KMP_LOCK_SHIFT) 1195 extern void __kmp_init_dynamic_user_locks();
1198 extern kmp_indirect_lock_t * __kmp_allocate_indirect_lock(
void **, kmp_int32, kmp_indirect_locktag_t);
1201 extern void __kmp_cleanup_indirect_user_locks();
1204 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
1207 extern void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *);
1208 #define KMP_SET_I_LOCK_LOCATION(lck, loc) { \ 1209 if (__kmp_indirect_set_location[(lck)->type] != NULL) \ 1210 __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \ 1214 extern void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t);
1215 #define KMP_SET_I_LOCK_FLAGS(lck, flag) { \ 1216 if (__kmp_indirect_set_flags[(lck)->type] != NULL) \ 1217 __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \ 1221 extern const ident_t * (*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p);
1222 #define KMP_GET_I_LOCK_LOCATION(lck) ( __kmp_indirect_get_location[(lck)->type] != NULL \ 1223 ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \ 1227 extern kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p);
1228 #define KMP_GET_I_LOCK_FLAGS(lck) ( __kmp_indirect_get_flags[(lck)->type] != NULL \ 1229 ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \ 1232 #define KMP_I_LOCK_CHUNK 1024 // number of kmp_indirect_lock_t objects to be allocated together 1235 typedef struct kmp_indirect_lock_table {
1236 kmp_indirect_lock_t **table;
1237 kmp_lock_index_t size;
1238 kmp_lock_index_t next;
1239 } kmp_indirect_lock_table_t;
1241 extern kmp_indirect_lock_table_t __kmp_i_lock_table;
1244 #define KMP_GET_I_LOCK(index) (*(__kmp_i_lock_table.table + (index)/KMP_I_LOCK_CHUNK) + (index)%KMP_I_LOCK_CHUNK) 1249 extern int __kmp_num_locks_in_block;
1252 #define KMP_LOOKUP_I_LOCK(l) ( (OMP_LOCK_T_SIZE < sizeof(void *)) \ 1253 ? KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(l)) \ 1254 : *((kmp_indirect_lock_t **)(l)) ) 1258 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
1260 #else // KMP_USE_DYNAMIC_LOCK 1262 # define KMP_LOCK_BUSY(v, type) (v) 1263 # define KMP_LOCK_FREE(type) 0 1264 # define KMP_LOCK_STRIP(v) (v) 1266 #endif // KMP_USE_DYNAMIC_LOCK 1270 #endif // __cplusplus