20 #include "kmp_debug.h" 33 #define KMP_PAD(type, sz) \ 34 (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1)) 35 #define KMP_GTID_DNE (-2) 54 #if KMP_OS_LINUX && defined(KMP_GOMP_COMPAT) 55 #define OMP_LOCK_T_SIZE sizeof(int) 56 #define OMP_NEST_LOCK_T_SIZE sizeof(void *) 58 #define OMP_LOCK_T_SIZE sizeof(void *) 59 #define OMP_NEST_LOCK_T_SIZE sizeof(void *) 65 #define OMP_CRITICAL_SIZE sizeof(void *) 66 #define INTEL_CRITICAL_SIZE 32 69 typedef kmp_uint32 kmp_lock_flags_t;
71 #define kmp_lf_critical_section 1 74 typedef kmp_uint32 kmp_lock_index_t;
78 struct kmp_lock_pool {
79 union kmp_user_lock *next;
80 kmp_lock_index_t index;
83 typedef struct kmp_lock_pool kmp_lock_pool_t;
85 extern void __kmp_validate_locks(
void);
122 struct kmp_base_tas_lock {
124 std::atomic<kmp_int32> poll;
125 kmp_int32 depth_locked;
128 typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
131 kmp_base_tas_lock_t lk;
132 kmp_lock_pool_t pool;
136 typedef union kmp_tas_lock kmp_tas_lock_t;
140 #define KMP_TAS_LOCK_INITIALIZER(lock) \ 142 { ATOMIC_VAR_INIT(KMP_LOCK_FREE(tas)), 0 } \ 145 extern int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
146 extern int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
147 extern int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
148 extern void __kmp_init_tas_lock(kmp_tas_lock_t *lck);
149 extern void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck);
151 extern int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
152 extern int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
153 extern int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
154 extern void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck);
155 extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
157 #define KMP_LOCK_RELEASED 1 158 #define KMP_LOCK_STILL_HELD 0 159 #define KMP_LOCK_ACQUIRED_FIRST 1 160 #define KMP_LOCK_ACQUIRED_NEXT 0 161 #ifndef KMP_USE_FUTEX 162 #define KMP_USE_FUTEX \ 163 (KMP_OS_LINUX && !KMP_OS_CNK && \ 164 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)) 179 struct kmp_base_futex_lock {
180 volatile kmp_int32 poll;
183 kmp_int32 depth_locked;
186 typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
188 union kmp_futex_lock {
189 kmp_base_futex_lock_t lk;
190 kmp_lock_pool_t pool;
195 typedef union kmp_futex_lock kmp_futex_lock_t;
199 #define KMP_FUTEX_LOCK_INITIALIZER(lock) \ 201 { KMP_LOCK_FREE(futex), 0 } \ 204 extern int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
205 extern int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
206 extern int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
207 extern void __kmp_init_futex_lock(kmp_futex_lock_t *lck);
208 extern void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck);
210 extern int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck,
212 extern int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
213 extern int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck,
215 extern void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck);
216 extern void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck);
218 #endif // KMP_USE_FUTEX 229 struct kmp_base_ticket_lock {
231 std::atomic_bool initialized;
232 volatile union kmp_ticket_lock *
self;
236 std::atomic_uint now_serving;
237 std::atomic_int owner_id;
238 std::atomic_int depth_locked;
239 kmp_lock_flags_t flags;
242 struct kmp_base_ticket_lock {
244 std::atomic<bool> initialized;
245 volatile union kmp_ticket_lock *
self;
247 std::atomic<unsigned>
249 std::atomic<unsigned>
251 std::atomic<int> owner_id;
252 std::atomic<int> depth_locked;
253 kmp_lock_flags_t flags;
259 struct kmp_base_ticket_lock;
261 #endif // !__cplusplus 263 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
265 union KMP_ALIGN_CACHE kmp_ticket_lock {
266 kmp_base_ticket_lock_t
268 kmp_lock_pool_t pool;
270 char lk_pad[KMP_PAD(kmp_base_ticket_lock_t, CACHE_LINE)];
273 typedef union kmp_ticket_lock kmp_ticket_lock_t;
278 #define KMP_TICKET_LOCK_INITIALIZER(lock) \ 281 ATOMIC_VAR_INIT(true) \ 282 , &(lock), NULL, ATOMIC_VAR_INIT(0U), ATOMIC_VAR_INIT(0U), \ 283 ATOMIC_VAR_INIT(0), ATOMIC_VAR_INIT(-1) \ 287 extern int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
288 extern int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
289 extern int __kmp_test_ticket_lock_with_cheks(kmp_ticket_lock_t *lck,
291 extern int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
292 extern void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck);
293 extern void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck);
295 extern int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck,
297 extern int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck,
299 extern int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck,
301 extern void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck);
302 extern void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck);
307 #if KMP_USE_ADAPTIVE_LOCKS 309 struct kmp_adaptive_lock_info;
311 typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
313 #if KMP_DEBUG_ADAPTIVE_LOCKS 315 struct kmp_adaptive_lock_statistics {
317 kmp_adaptive_lock_info_t *next;
318 kmp_adaptive_lock_info_t *prev;
321 kmp_uint32 successfulSpeculations;
322 kmp_uint32 hardFailedSpeculations;
323 kmp_uint32 softFailedSpeculations;
324 kmp_uint32 nonSpeculativeAcquires;
325 kmp_uint32 nonSpeculativeAcquireAttempts;
326 kmp_uint32 lemmingYields;
329 typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
331 extern void __kmp_print_speculative_stats();
332 extern void __kmp_init_speculative_stats();
334 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 336 struct kmp_adaptive_lock_info {
341 kmp_uint32
volatile badness;
342 kmp_uint32
volatile acquire_attempts;
344 kmp_uint32 max_badness;
345 kmp_uint32 max_soft_retries;
347 #if KMP_DEBUG_ADAPTIVE_LOCKS 348 kmp_adaptive_lock_statistics_t
volatile stats;
352 #endif // KMP_USE_ADAPTIVE_LOCKS 354 struct kmp_base_queuing_lock {
357 volatile union kmp_queuing_lock
375 volatile kmp_int32 owner_id;
376 kmp_int32 depth_locked;
378 kmp_lock_flags_t flags;
381 typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
383 KMP_BUILD_ASSERT(offsetof(kmp_base_queuing_lock_t, tail_id) % 8 == 0);
385 union KMP_ALIGN_CACHE kmp_queuing_lock {
386 kmp_base_queuing_lock_t
388 kmp_lock_pool_t pool;
390 char lk_pad[KMP_PAD(kmp_base_queuing_lock_t, CACHE_LINE)];
393 typedef union kmp_queuing_lock kmp_queuing_lock_t;
395 extern int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
396 extern int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
397 extern int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
398 extern void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck);
399 extern void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck);
401 extern int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck,
403 extern int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck,
405 extern int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck,
407 extern void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck);
408 extern void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck);
410 #if KMP_USE_ADAPTIVE_LOCKS 414 struct kmp_base_adaptive_lock {
415 kmp_base_queuing_lock qlk;
416 KMP_ALIGN(CACHE_LINE)
417 kmp_adaptive_lock_info_t
421 typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
423 union KMP_ALIGN_CACHE kmp_adaptive_lock {
424 kmp_base_adaptive_lock_t lk;
425 kmp_lock_pool_t pool;
427 char lk_pad[KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE)];
429 typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
431 #define GET_QLK_PTR(l) ((kmp_queuing_lock_t *)&(l)->lk.qlk) 433 #endif // KMP_USE_ADAPTIVE_LOCKS 437 struct kmp_base_drdpa_lock {
446 volatile union kmp_drdpa_lock
449 std::atomic<std::atomic<kmp_uint64> *> polls;
450 std::atomic<kmp_uint64> mask;
451 kmp_uint64 cleanup_ticket;
452 std::atomic<kmp_uint64> *old_polls;
453 kmp_uint32 num_polls;
459 std::atomic<kmp_uint64> next_ticket;
474 kmp_uint64 now_serving;
475 volatile kmp_uint32 owner_id;
476 kmp_int32 depth_locked;
477 kmp_lock_flags_t flags;
480 typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
482 union KMP_ALIGN_CACHE kmp_drdpa_lock {
483 kmp_base_drdpa_lock_t
485 kmp_lock_pool_t pool;
487 char lk_pad[KMP_PAD(kmp_base_drdpa_lock_t, CACHE_LINE)];
490 typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
492 extern int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
493 extern int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
494 extern int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
495 extern void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck);
496 extern void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck);
498 extern int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
500 extern int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
501 extern int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
503 extern void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
504 extern void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
518 typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
520 #define KMP_BOOTSTRAP_LOCK_INITIALIZER(lock) KMP_TICKET_LOCK_INITIALIZER((lock)) 521 #define KMP_BOOTSTRAP_LOCK_INIT(lock) \ 522 kmp_bootstrap_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock) 524 static inline int __kmp_acquire_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
525 return __kmp_acquire_ticket_lock(lck, KMP_GTID_DNE);
528 static inline int __kmp_test_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
529 return __kmp_test_ticket_lock(lck, KMP_GTID_DNE);
532 static inline void __kmp_release_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
533 __kmp_release_ticket_lock(lck, KMP_GTID_DNE);
536 static inline void __kmp_init_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
537 __kmp_init_ticket_lock(lck);
540 static inline void __kmp_destroy_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
541 __kmp_destroy_ticket_lock(lck);
552 typedef kmp_ticket_lock_t kmp_lock_t;
554 #define KMP_LOCK_INIT(lock) kmp_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock) 556 static inline int __kmp_acquire_lock(kmp_lock_t *lck, kmp_int32 gtid) {
557 return __kmp_acquire_ticket_lock(lck, gtid);
560 static inline int __kmp_test_lock(kmp_lock_t *lck, kmp_int32 gtid) {
561 return __kmp_test_ticket_lock(lck, gtid);
564 static inline void __kmp_release_lock(kmp_lock_t *lck, kmp_int32 gtid) {
565 __kmp_release_ticket_lock(lck, gtid);
568 static inline void __kmp_init_lock(kmp_lock_t *lck) {
569 __kmp_init_ticket_lock(lck);
572 static inline void __kmp_destroy_lock(kmp_lock_t *lck) {
573 __kmp_destroy_ticket_lock(lck);
589 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX 596 #if KMP_USE_ADAPTIVE_LOCKS 598 #endif // KMP_USE_ADAPTIVE_LOCKS 601 typedef enum kmp_lock_kind kmp_lock_kind_t;
603 extern kmp_lock_kind_t __kmp_user_lock_kind;
605 union kmp_user_lock {
608 kmp_futex_lock_t futex;
610 kmp_ticket_lock_t ticket;
611 kmp_queuing_lock_t queuing;
612 kmp_drdpa_lock_t drdpa;
613 #if KMP_USE_ADAPTIVE_LOCKS 614 kmp_adaptive_lock_t adaptive;
615 #endif // KMP_USE_ADAPTIVE_LOCKS 616 kmp_lock_pool_t pool;
619 typedef union kmp_user_lock *kmp_user_lock_p;
621 #if !KMP_USE_DYNAMIC_LOCK 623 extern size_t __kmp_base_user_lock_size;
624 extern size_t __kmp_user_lock_size;
626 extern kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck);
628 static inline kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck) {
629 KMP_DEBUG_ASSERT(__kmp_get_user_lock_owner_ != NULL);
630 return (*__kmp_get_user_lock_owner_)(lck);
633 extern int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
636 #if KMP_OS_LINUX && \ 637 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 639 #define __kmp_acquire_user_lock_with_checks(lck, gtid) \ 640 if (__kmp_user_lock_kind == lk_tas) { \ 641 if (__kmp_env_consistency_check) { \ 642 char const *const func = "omp_set_lock"; \ 643 if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) && \ 644 lck->tas.lk.depth_locked != -1) { \ 645 KMP_FATAL(LockNestableUsedAsSimple, func); \ 647 if ((gtid >= 0) && (lck->tas.lk.poll - 1 == gtid)) { \ 648 KMP_FATAL(LockIsAlreadyOwned, func); \ 651 if (lck->tas.lk.poll != 0 || \ 652 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \ 654 KMP_FSYNC_PREPARE(lck); \ 655 KMP_INIT_YIELD(spins); \ 656 if (TCR_4(__kmp_nth) > \ 657 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 660 KMP_YIELD_SPIN(spins); \ 662 while (lck->tas.lk.poll != 0 || !__kmp_atomic_compare_store_acq( \ 663 &lck->tas.lk.poll, 0, gtid + 1)) { \ 664 if (TCR_4(__kmp_nth) > \ 665 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 668 KMP_YIELD_SPIN(spins); \ 672 KMP_FSYNC_ACQUIRED(lck); \ 674 KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL); \ 675 (*__kmp_acquire_user_lock_with_checks_)(lck, gtid); \ 679 static inline int __kmp_acquire_user_lock_with_checks(kmp_user_lock_p lck,
681 KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL);
682 return (*__kmp_acquire_user_lock_with_checks_)(lck, gtid);
686 extern int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
689 #if KMP_OS_LINUX && \ 690 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 692 #include "kmp_i18n.h" 693 extern int __kmp_env_consistency_check;
694 static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
696 if (__kmp_user_lock_kind == lk_tas) {
697 if (__kmp_env_consistency_check) {
698 char const *
const func =
"omp_test_lock";
699 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
700 lck->tas.lk.depth_locked != -1) {
701 KMP_FATAL(LockNestableUsedAsSimple, func);
704 return ((lck->tas.lk.poll == 0) &&
705 __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
707 KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
708 return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
712 static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
714 KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
715 return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
719 extern int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
722 static inline void __kmp_release_user_lock_with_checks(kmp_user_lock_p lck,
724 KMP_DEBUG_ASSERT(__kmp_release_user_lock_with_checks_ != NULL);
725 (*__kmp_release_user_lock_with_checks_)(lck, gtid);
728 extern void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck);
730 static inline void __kmp_init_user_lock_with_checks(kmp_user_lock_p lck) {
731 KMP_DEBUG_ASSERT(__kmp_init_user_lock_with_checks_ != NULL);
732 (*__kmp_init_user_lock_with_checks_)(lck);
737 extern void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck);
739 static inline void __kmp_destroy_user_lock(kmp_user_lock_p lck) {
740 KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_ != NULL);
741 (*__kmp_destroy_user_lock_)(lck);
744 extern void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck);
746 static inline void __kmp_destroy_user_lock_with_checks(kmp_user_lock_p lck) {
747 KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_with_checks_ != NULL);
748 (*__kmp_destroy_user_lock_with_checks_)(lck);
751 extern int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
754 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 756 #define __kmp_acquire_nested_user_lock_with_checks(lck, gtid, depth) \ 757 if (__kmp_user_lock_kind == lk_tas) { \ 758 if (__kmp_env_consistency_check) { \ 759 char const *const func = "omp_set_nest_lock"; \ 760 if ((sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) && \ 761 lck->tas.lk.depth_locked == -1) { \ 762 KMP_FATAL(LockSimpleUsedAsNestable, func); \ 765 if (lck->tas.lk.poll - 1 == gtid) { \ 766 lck->tas.lk.depth_locked += 1; \ 767 *depth = KMP_LOCK_ACQUIRED_NEXT; \ 769 if ((lck->tas.lk.poll != 0) || \ 770 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \ 772 KMP_FSYNC_PREPARE(lck); \ 773 KMP_INIT_YIELD(spins); \ 774 if (TCR_4(__kmp_nth) > \ 775 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 778 KMP_YIELD_SPIN(spins); \ 781 (lck->tas.lk.poll != 0) || \ 782 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \ 783 if (TCR_4(__kmp_nth) > \ 784 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ 787 KMP_YIELD_SPIN(spins); \ 791 lck->tas.lk.depth_locked = 1; \ 792 *depth = KMP_LOCK_ACQUIRED_FIRST; \ 794 KMP_FSYNC_ACQUIRED(lck); \ 796 KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL); \ 797 *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid); \ 802 __kmp_acquire_nested_user_lock_with_checks(kmp_user_lock_p lck, kmp_int32 gtid,
804 KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL);
805 *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid);
809 extern int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
812 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 813 static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
815 if (__kmp_user_lock_kind == lk_tas) {
817 if (__kmp_env_consistency_check) {
818 char const *
const func =
"omp_test_nest_lock";
819 if ((
sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) &&
820 lck->tas.lk.depth_locked == -1) {
821 KMP_FATAL(LockSimpleUsedAsNestable, func);
824 KMP_DEBUG_ASSERT(gtid >= 0);
825 if (lck->tas.lk.poll - 1 ==
827 return ++lck->tas.lk.depth_locked;
829 retval = ((lck->tas.lk.poll == 0) &&
830 __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
833 lck->tas.lk.depth_locked = 1;
837 KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
838 return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
842 static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
844 KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
845 return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
849 extern int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
853 __kmp_release_nested_user_lock_with_checks(kmp_user_lock_p lck,
855 KMP_DEBUG_ASSERT(__kmp_release_nested_user_lock_with_checks_ != NULL);
856 return (*__kmp_release_nested_user_lock_with_checks_)(lck, gtid);
859 extern void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
862 __kmp_init_nested_user_lock_with_checks(kmp_user_lock_p lck) {
863 KMP_DEBUG_ASSERT(__kmp_init_nested_user_lock_with_checks_ != NULL);
864 (*__kmp_init_nested_user_lock_with_checks_)(lck);
867 extern void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
870 __kmp_destroy_nested_user_lock_with_checks(kmp_user_lock_p lck) {
871 KMP_DEBUG_ASSERT(__kmp_destroy_nested_user_lock_with_checks_ != NULL);
872 (*__kmp_destroy_nested_user_lock_with_checks_)(lck);
888 extern int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck);
892 extern const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck);
894 static inline const ident_t *__kmp_get_user_lock_location(kmp_user_lock_p lck) {
895 if (__kmp_get_user_lock_location_ != NULL) {
896 return (*__kmp_get_user_lock_location_)(lck);
902 extern void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
905 static inline void __kmp_set_user_lock_location(kmp_user_lock_p lck,
907 if (__kmp_set_user_lock_location_ != NULL) {
908 (*__kmp_set_user_lock_location_)(lck, loc);
912 extern kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck);
914 extern void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
915 kmp_lock_flags_t flags);
917 static inline void __kmp_set_user_lock_flags(kmp_user_lock_p lck,
918 kmp_lock_flags_t flags) {
919 if (__kmp_set_user_lock_flags_ != NULL) {
920 (*__kmp_set_user_lock_flags_)(lck, flags);
925 extern void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind);
928 #define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) \ 930 __kmp_acquire##nest##user_lock_with_checks_ = (int (*)( \ 931 kmp_user_lock_p, kmp_int32))__kmp_acquire##nest##kind##_##suffix; \ 932 __kmp_release##nest##user_lock_with_checks_ = (int (*)( \ 933 kmp_user_lock_p, kmp_int32))__kmp_release##nest##kind##_##suffix; \ 934 __kmp_test##nest##user_lock_with_checks_ = (int (*)( \ 935 kmp_user_lock_p, kmp_int32))__kmp_test##nest##kind##_##suffix; \ 936 __kmp_init##nest##user_lock_with_checks_ = \ 937 (void (*)(kmp_user_lock_p))__kmp_init##nest##kind##_##suffix; \ 938 __kmp_destroy##nest##user_lock_with_checks_ = \ 939 (void (*)(kmp_user_lock_p))__kmp_destroy##nest##kind##_##suffix; \ 942 #define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock) 943 #define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) \ 944 KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks) 945 #define KMP_BIND_NESTED_USER_LOCK(kind) \ 946 KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock) 947 #define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) \ 948 KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks) 973 struct kmp_lock_table {
974 kmp_lock_index_t used;
975 kmp_lock_index_t allocated;
976 kmp_user_lock_p *table;
979 typedef struct kmp_lock_table kmp_lock_table_t;
981 extern kmp_lock_table_t __kmp_user_lock_table;
982 extern kmp_user_lock_p __kmp_lock_pool;
984 struct kmp_block_of_locks {
985 struct kmp_block_of_locks *next_block;
989 typedef struct kmp_block_of_locks kmp_block_of_locks_t;
991 extern kmp_block_of_locks_t *__kmp_lock_blocks;
992 extern int __kmp_num_locks_in_block;
994 extern kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock,
996 kmp_lock_flags_t flags);
997 extern void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
998 kmp_user_lock_p lck);
999 extern kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
1001 extern void __kmp_cleanup_user_locks();
1003 #define KMP_CHECK_USER_LOCK_INIT() \ 1005 if (!TCR_4(__kmp_init_user_locks)) { \ 1006 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); \ 1007 if (!TCR_4(__kmp_init_user_locks)) { \ 1008 TCW_4(__kmp_init_user_locks, TRUE); \ 1010 __kmp_release_bootstrap_lock(&__kmp_initz_lock); \ 1014 #endif // KMP_USE_DYNAMIC_LOCK 1019 #if KMP_USE_DYNAMIC_LOCK 1057 #define KMP_USE_INLINED_TAS \ 1058 (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1 1059 #define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0 1066 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) 1067 #define KMP_FOREACH_I_LOCK(m, a) \ 1068 m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \ 1069 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1070 m(nested_queuing, a) m(nested_drdpa, a) 1072 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) 1073 #define KMP_FOREACH_I_LOCK(m, a) \ 1074 m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \ 1075 m(nested_tas, a) m(nested_ticket, a) m(nested_queuing, a) \ 1077 #endif // KMP_USE_FUTEX 1078 #define KMP_LAST_D_LOCK lockseq_hle 1081 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) 1082 #define KMP_FOREACH_I_LOCK(m, a) \ 1083 m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_futex, a) \ 1084 m(nested_ticket, a) m(nested_queuing, a) m(nested_drdpa, a) 1085 #define KMP_LAST_D_LOCK lockseq_futex 1087 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) 1088 #define KMP_FOREACH_I_LOCK(m, a) \ 1089 m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_ticket, a) \ 1090 m(nested_queuing, a) m(nested_drdpa, a) 1091 #define KMP_LAST_D_LOCK lockseq_tas 1092 #endif // KMP_USE_FUTEX 1093 #endif // KMP_USE_TSX 1096 #define KMP_LOCK_SHIFT \ 1097 8 // number of low bits to be used as tag for direct locks 1098 #define KMP_FIRST_D_LOCK lockseq_tas 1099 #define KMP_FIRST_I_LOCK lockseq_ticket 1100 #define KMP_LAST_I_LOCK lockseq_nested_drdpa 1101 #define KMP_NUM_I_LOCKS \ 1102 (locktag_nested_drdpa + 1) // number of indirect lock types 1105 typedef kmp_uint32 kmp_dyna_lock_t;
1110 lockseq_indirect = 0,
1111 #define expand_seq(l, a) lockseq_##l, 1112 KMP_FOREACH_D_LOCK(expand_seq, 0) KMP_FOREACH_I_LOCK(expand_seq, 0)
1114 } kmp_dyna_lockseq_t;
1118 #define expand_tag(l, a) locktag_##l, 1119 KMP_FOREACH_I_LOCK(expand_tag, 0)
1121 } kmp_indirect_locktag_t;
1124 #define KMP_IS_D_LOCK(seq) \ 1125 ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK) 1126 #define KMP_IS_I_LOCK(seq) \ 1127 ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK) 1128 #define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq)-KMP_FIRST_I_LOCK) 1129 #define KMP_GET_D_TAG(seq) ((seq) << 1 | 1) 1133 #define expand_tag(l, a) locktag_##l = KMP_GET_D_TAG(lockseq_##l), 1134 KMP_FOREACH_D_LOCK(expand_tag, 0)
1136 } kmp_direct_locktag_t;
1140 kmp_user_lock_p lock;
1141 kmp_indirect_locktag_t type;
1142 } kmp_indirect_lock_t;
1146 extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
1147 extern void (*(*__kmp_direct_destroy))(kmp_dyna_lock_t *);
1148 extern int (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32);
1149 extern int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32);
1150 extern int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32);
1154 extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
1155 extern void (*(*__kmp_indirect_destroy))(kmp_user_lock_p);
1156 extern int (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32);
1157 extern int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32);
1158 extern int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32);
1161 #define KMP_EXTRACT_D_TAG(l) \ 1162 (*((kmp_dyna_lock_t *)(l)) & ((1 << KMP_LOCK_SHIFT) - 1) & \ 1163 -(*((kmp_dyna_lock_t *)(l)) & 1)) 1166 #define KMP_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1) 1170 #define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)] 1174 #define KMP_I_LOCK_FUNC(l, op) \ 1175 __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type] 1178 #define KMP_INIT_D_LOCK(l, seq) \ 1179 __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq) 1182 #define KMP_INIT_I_LOCK(l, seq) \ 1183 __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq) 1186 #define KMP_LOCK_FREE(type) (locktag_##type) 1189 #define KMP_LOCK_BUSY(v, type) ((v) << KMP_LOCK_SHIFT | locktag_##type) 1192 #define KMP_LOCK_STRIP(v) ((v) >> KMP_LOCK_SHIFT) 1196 extern void __kmp_init_dynamic_user_locks();
1199 extern kmp_indirect_lock_t *
1200 __kmp_allocate_indirect_lock(
void **, kmp_int32, kmp_indirect_locktag_t);
1203 extern void __kmp_cleanup_indirect_user_locks();
1206 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
1209 extern void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
1211 #define KMP_SET_I_LOCK_LOCATION(lck, loc) \ 1213 if (__kmp_indirect_set_location[(lck)->type] != NULL) \ 1214 __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \ 1218 extern void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
1220 #define KMP_SET_I_LOCK_FLAGS(lck, flag) \ 1222 if (__kmp_indirect_set_flags[(lck)->type] != NULL) \ 1223 __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \ 1227 extern const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
1229 #define KMP_GET_I_LOCK_LOCATION(lck) \ 1230 (__kmp_indirect_get_location[(lck)->type] != NULL \ 1231 ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \ 1235 extern kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
1237 #define KMP_GET_I_LOCK_FLAGS(lck) \ 1238 (__kmp_indirect_get_flags[(lck)->type] != NULL \ 1239 ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \ 1242 #define KMP_I_LOCK_CHUNK \ 1243 1024 // number of kmp_indirect_lock_t objects to be allocated together 1246 typedef struct kmp_indirect_lock_table {
1247 kmp_indirect_lock_t **table;
1248 kmp_lock_index_t size;
1249 kmp_lock_index_t next;
1250 } kmp_indirect_lock_table_t;
1252 extern kmp_indirect_lock_table_t __kmp_i_lock_table;
1255 #define KMP_GET_I_LOCK(index) \ 1256 (*(__kmp_i_lock_table.table + (index) / KMP_I_LOCK_CHUNK) + \ 1257 (index) % KMP_I_LOCK_CHUNK) 1262 extern int __kmp_num_locks_in_block;
1265 #define KMP_LOOKUP_I_LOCK(l) \ 1266 ((OMP_LOCK_T_SIZE < sizeof(void *)) ? KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(l)) \ 1267 : *((kmp_indirect_lock_t **)(l))) 1270 extern kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
1272 #else // KMP_USE_DYNAMIC_LOCK 1274 #define KMP_LOCK_BUSY(v, type) (v) 1275 #define KMP_LOCK_FREE(type) 0 1276 #define KMP_LOCK_STRIP(v) (v) 1278 #endif // KMP_USE_DYNAMIC_LOCK 1283 kmp_uint32 max_backoff;
1284 kmp_uint32 min_tick;
1288 extern kmp_backoff_t __kmp_spin_backoff_params;
1291 extern void __kmp_spin_backoff(kmp_backoff_t *);
1295 #endif // __cplusplus