24 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 26 # include <sys/syscall.h> 44 __kmp_validate_locks(
void )
50 x = ~((kmp_uint32) 0) - 2;
53 for (i = 0; i < 8; ++i, ++x, ++y) {
54 kmp_uint32 z = (x - y);
58 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
76 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
78 return KMP_LOCK_STRIP(TCR_4( lck->lk.poll )) - 1;
82 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
84 return lck->lk.depth_locked != -1;
87 __forceinline
static int 88 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
92 #ifdef USE_LOCK_PROFILE 93 kmp_uint32 curr = TCR_4( lck->lk.poll );
94 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
95 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
99 if ( ( lck->lk.poll == KMP_LOCK_FREE(tas) )
100 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) {
101 KMP_FSYNC_ACQUIRED(lck);
102 return KMP_LOCK_ACQUIRED_FIRST;
106 KMP_FSYNC_PREPARE( lck );
107 KMP_INIT_YIELD( spins );
108 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
113 KMP_YIELD_SPIN( spins );
116 while ( ( lck->lk.poll != KMP_LOCK_FREE(tas) ) ||
117 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) ) {
121 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
126 KMP_YIELD_SPIN( spins );
129 KMP_FSYNC_ACQUIRED( lck );
130 return KMP_LOCK_ACQUIRED_FIRST;
134 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
136 return __kmp_acquire_tas_lock_timed_template( lck, gtid );
140 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
142 char const *
const func =
"omp_set_lock";
143 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
144 && __kmp_is_tas_lock_nestable( lck ) ) {
145 KMP_FATAL( LockNestableUsedAsSimple, func );
147 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
148 KMP_FATAL( LockIsAlreadyOwned, func );
150 return __kmp_acquire_tas_lock( lck, gtid );
154 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
156 if ( ( lck->lk.poll == KMP_LOCK_FREE(tas) )
157 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) {
158 KMP_FSYNC_ACQUIRED( lck );
165 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
167 char const *
const func =
"omp_test_lock";
168 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
169 && __kmp_is_tas_lock_nestable( lck ) ) {
170 KMP_FATAL( LockNestableUsedAsSimple, func );
172 return __kmp_test_tas_lock( lck, gtid );
176 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
180 KMP_FSYNC_RELEASING(lck);
181 KMP_ST_REL32( &(lck->lk.poll), KMP_LOCK_FREE(tas) );
184 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
186 return KMP_LOCK_RELEASED;
190 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
192 char const *
const func =
"omp_unset_lock";
194 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
195 && __kmp_is_tas_lock_nestable( lck ) ) {
196 KMP_FATAL( LockNestableUsedAsSimple, func );
198 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
199 KMP_FATAL( LockUnsettingFree, func );
201 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
202 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
203 KMP_FATAL( LockUnsettingSetByAnother, func );
205 return __kmp_release_tas_lock( lck, gtid );
209 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
211 TCW_4( lck->lk.poll, KMP_LOCK_FREE(tas) );
215 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
217 __kmp_init_tas_lock( lck );
221 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
227 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
229 char const *
const func =
"omp_destroy_lock";
230 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
231 && __kmp_is_tas_lock_nestable( lck ) ) {
232 KMP_FATAL( LockNestableUsedAsSimple, func );
234 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
235 KMP_FATAL( LockStillOwned, func );
237 __kmp_destroy_tas_lock( lck );
246 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
248 KMP_DEBUG_ASSERT( gtid >= 0 );
250 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
251 lck->lk.depth_locked += 1;
252 return KMP_LOCK_ACQUIRED_NEXT;
255 __kmp_acquire_tas_lock_timed_template( lck, gtid );
256 lck->lk.depth_locked = 1;
257 return KMP_LOCK_ACQUIRED_FIRST;
262 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
264 char const *
const func =
"omp_set_nest_lock";
265 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
266 KMP_FATAL( LockSimpleUsedAsNestable, func );
268 return __kmp_acquire_nested_tas_lock( lck, gtid );
272 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
276 KMP_DEBUG_ASSERT( gtid >= 0 );
278 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
279 retval = ++lck->lk.depth_locked;
281 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
286 retval = lck->lk.depth_locked = 1;
292 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
294 char const *
const func =
"omp_test_nest_lock";
295 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
296 KMP_FATAL( LockSimpleUsedAsNestable, func );
298 return __kmp_test_nested_tas_lock( lck, gtid );
302 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
304 KMP_DEBUG_ASSERT( gtid >= 0 );
307 if ( --(lck->lk.depth_locked) == 0 ) {
308 __kmp_release_tas_lock( lck, gtid );
309 return KMP_LOCK_RELEASED;
311 return KMP_LOCK_STILL_HELD;
315 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
317 char const *
const func =
"omp_unset_nest_lock";
319 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
320 KMP_FATAL( LockSimpleUsedAsNestable, func );
322 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
323 KMP_FATAL( LockUnsettingFree, func );
325 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
326 KMP_FATAL( LockUnsettingSetByAnother, func );
328 return __kmp_release_nested_tas_lock( lck, gtid );
332 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
334 __kmp_init_tas_lock( lck );
335 lck->lk.depth_locked = 0;
339 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
341 __kmp_init_nested_tas_lock( lck );
345 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
347 __kmp_destroy_tas_lock( lck );
348 lck->lk.depth_locked = 0;
352 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
354 char const *
const func =
"omp_destroy_nest_lock";
355 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
356 KMP_FATAL( LockSimpleUsedAsNestable, func );
358 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
359 KMP_FATAL( LockStillOwned, func );
361 __kmp_destroy_nested_tas_lock( lck );
365 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 376 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
378 return KMP_LOCK_STRIP(( TCR_4( lck->lk.poll ) >> 1 )) - 1;
382 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
384 return lck->lk.depth_locked != -1;
387 __forceinline
static int 388 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
390 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
394 #ifdef USE_LOCK_PROFILE 395 kmp_uint32 curr = TCR_4( lck->lk.poll );
396 if ( ( curr != 0 ) && ( curr != gtid_code ) )
397 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
401 KMP_FSYNC_PREPARE( lck );
402 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
403 lck, lck->lk.poll, gtid ) );
407 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex),
408 KMP_LOCK_BUSY(gtid_code, futex) ) ) != KMP_LOCK_FREE(futex) ) {
410 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
411 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
412 lck, gtid, poll_val, cond ) );
427 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ), poll_val, poll_val | KMP_LOCK_BUSY(1, futex) ) ) {
428 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
429 lck, lck->lk.poll, gtid ) );
432 poll_val |= KMP_LOCK_BUSY(1, futex);
434 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
435 lck, lck->lk.poll, gtid ) );
438 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
439 lck, gtid, poll_val ) );
442 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
443 poll_val, NULL, NULL, 0 ) ) != 0 ) {
444 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
445 lck, gtid, poll_val, rc, errno ) );
449 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
450 lck, gtid, poll_val ) );
460 KMP_FSYNC_ACQUIRED( lck );
461 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
462 lck, lck->lk.poll, gtid ) );
463 return KMP_LOCK_ACQUIRED_FIRST;
467 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
469 return __kmp_acquire_futex_lock_timed_template( lck, gtid );
473 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
475 char const *
const func =
"omp_set_lock";
476 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
477 && __kmp_is_futex_lock_nestable( lck ) ) {
478 KMP_FATAL( LockNestableUsedAsSimple, func );
480 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
481 KMP_FATAL( LockIsAlreadyOwned, func );
483 return __kmp_acquire_futex_lock( lck, gtid );
487 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
489 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1, futex) << 1 ) ) {
490 KMP_FSYNC_ACQUIRED( lck );
497 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
499 char const *
const func =
"omp_test_lock";
500 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
501 && __kmp_is_futex_lock_nestable( lck ) ) {
502 KMP_FATAL( LockNestableUsedAsSimple, func );
504 return __kmp_test_futex_lock( lck, gtid );
508 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
512 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
513 lck, lck->lk.poll, gtid ) );
515 KMP_FSYNC_RELEASING(lck);
517 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex) );
519 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
520 lck, gtid, poll_val ) );
522 if ( KMP_LOCK_STRIP(poll_val) & 1 ) {
523 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
525 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), NULL, NULL, 0 );
530 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
531 lck, lck->lk.poll, gtid ) );
533 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
535 return KMP_LOCK_RELEASED;
539 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
541 char const *
const func =
"omp_unset_lock";
543 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
544 && __kmp_is_futex_lock_nestable( lck ) ) {
545 KMP_FATAL( LockNestableUsedAsSimple, func );
547 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
548 KMP_FATAL( LockUnsettingFree, func );
550 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
551 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
552 KMP_FATAL( LockUnsettingSetByAnother, func );
554 return __kmp_release_futex_lock( lck, gtid );
558 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
560 TCW_4( lck->lk.poll, KMP_LOCK_FREE(futex) );
564 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
566 __kmp_init_futex_lock( lck );
570 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
576 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
578 char const *
const func =
"omp_destroy_lock";
579 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
580 && __kmp_is_futex_lock_nestable( lck ) ) {
581 KMP_FATAL( LockNestableUsedAsSimple, func );
583 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
584 KMP_FATAL( LockStillOwned, func );
586 __kmp_destroy_futex_lock( lck );
595 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
597 KMP_DEBUG_ASSERT( gtid >= 0 );
599 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
600 lck->lk.depth_locked += 1;
601 return KMP_LOCK_ACQUIRED_NEXT;
604 __kmp_acquire_futex_lock_timed_template( lck, gtid );
605 lck->lk.depth_locked = 1;
606 return KMP_LOCK_ACQUIRED_FIRST;
611 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
613 char const *
const func =
"omp_set_nest_lock";
614 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
615 KMP_FATAL( LockSimpleUsedAsNestable, func );
617 return __kmp_acquire_nested_futex_lock( lck, gtid );
621 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
625 KMP_DEBUG_ASSERT( gtid >= 0 );
627 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
628 retval = ++lck->lk.depth_locked;
630 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
635 retval = lck->lk.depth_locked = 1;
641 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
643 char const *
const func =
"omp_test_nest_lock";
644 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
645 KMP_FATAL( LockSimpleUsedAsNestable, func );
647 return __kmp_test_nested_futex_lock( lck, gtid );
651 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
653 KMP_DEBUG_ASSERT( gtid >= 0 );
656 if ( --(lck->lk.depth_locked) == 0 ) {
657 __kmp_release_futex_lock( lck, gtid );
658 return KMP_LOCK_RELEASED;
660 return KMP_LOCK_STILL_HELD;
664 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
666 char const *
const func =
"omp_unset_nest_lock";
668 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
669 KMP_FATAL( LockSimpleUsedAsNestable, func );
671 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
672 KMP_FATAL( LockUnsettingFree, func );
674 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
675 KMP_FATAL( LockUnsettingSetByAnother, func );
677 return __kmp_release_nested_futex_lock( lck, gtid );
681 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
683 __kmp_init_futex_lock( lck );
684 lck->lk.depth_locked = 0;
688 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
690 __kmp_init_nested_futex_lock( lck );
694 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
696 __kmp_destroy_futex_lock( lck );
697 lck->lk.depth_locked = 0;
701 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
703 char const *
const func =
"omp_destroy_nest_lock";
704 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
705 KMP_FATAL( LockSimpleUsedAsNestable, func );
707 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
708 KMP_FATAL( LockStillOwned, func );
710 __kmp_destroy_nested_futex_lock( lck );
713 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) 720 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
722 return TCR_4( lck->lk.owner_id ) - 1;
726 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
728 return lck->lk.depth_locked != -1;
732 __kmp_bakery_check(kmp_uint32 value, kmp_uint32 checker)
734 register kmp_uint32 pause;
736 if (value == checker) {
739 for (pause = checker - value; pause != 0; --pause);
743 __forceinline
static int 744 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
746 kmp_uint32 my_ticket;
749 my_ticket = KMP_TEST_THEN_INC32( (kmp_int32 *) &lck->lk.next_ticket );
751 #ifdef USE_LOCK_PROFILE 752 if ( TCR_4( lck->lk.now_serving ) != my_ticket )
753 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
757 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
758 KMP_FSYNC_ACQUIRED(lck);
759 return KMP_LOCK_ACQUIRED_FIRST;
761 KMP_WAIT_YIELD( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
762 KMP_FSYNC_ACQUIRED(lck);
763 return KMP_LOCK_ACQUIRED_FIRST;
767 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
769 return __kmp_acquire_ticket_lock_timed_template( lck, gtid );
773 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
775 char const *
const func =
"omp_set_lock";
776 if ( lck->lk.initialized != lck ) {
777 KMP_FATAL( LockIsUninitialized, func );
779 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
780 KMP_FATAL( LockNestableUsedAsSimple, func );
782 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
783 KMP_FATAL( LockIsAlreadyOwned, func );
786 __kmp_acquire_ticket_lock( lck, gtid );
788 lck->lk.owner_id = gtid + 1;
789 return KMP_LOCK_ACQUIRED_FIRST;
793 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
795 kmp_uint32 my_ticket = TCR_4( lck->lk.next_ticket );
796 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
797 kmp_uint32 next_ticket = my_ticket + 1;
798 if ( KMP_COMPARE_AND_STORE_ACQ32( (kmp_int32 *) &lck->lk.next_ticket,
799 my_ticket, next_ticket ) ) {
800 KMP_FSYNC_ACQUIRED( lck );
808 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
810 char const *
const func =
"omp_test_lock";
811 if ( lck->lk.initialized != lck ) {
812 KMP_FATAL( LockIsUninitialized, func );
814 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
815 KMP_FATAL( LockNestableUsedAsSimple, func );
818 int retval = __kmp_test_ticket_lock( lck, gtid );
821 lck->lk.owner_id = gtid + 1;
827 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
833 KMP_FSYNC_RELEASING(lck);
834 distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) );
836 KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 );
841 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
842 return KMP_LOCK_RELEASED;
846 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
848 char const *
const func =
"omp_unset_lock";
850 if ( lck->lk.initialized != lck ) {
851 KMP_FATAL( LockIsUninitialized, func );
853 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
854 KMP_FATAL( LockNestableUsedAsSimple, func );
856 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
857 KMP_FATAL( LockUnsettingFree, func );
859 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
860 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
861 KMP_FATAL( LockUnsettingSetByAnother, func );
863 lck->lk.owner_id = 0;
864 return __kmp_release_ticket_lock( lck, gtid );
868 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
870 lck->lk.location = NULL;
871 TCW_4( lck->lk.next_ticket, 0 );
872 TCW_4( lck->lk.now_serving, 0 );
873 lck->lk.owner_id = 0;
874 lck->lk.depth_locked = -1;
875 lck->lk.initialized = (kmp_ticket_lock *)lck;
879 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
881 __kmp_init_ticket_lock( lck );
885 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
887 lck->lk.initialized = NULL;
888 lck->lk.location = NULL;
889 lck->lk.next_ticket = 0;
890 lck->lk.now_serving = 0;
891 lck->lk.owner_id = 0;
892 lck->lk.depth_locked = -1;
896 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
898 char const *
const func =
"omp_destroy_lock";
899 if ( lck->lk.initialized != lck ) {
900 KMP_FATAL( LockIsUninitialized, func );
902 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
903 KMP_FATAL( LockNestableUsedAsSimple, func );
905 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
906 KMP_FATAL( LockStillOwned, func );
908 __kmp_destroy_ticket_lock( lck );
917 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
919 KMP_DEBUG_ASSERT( gtid >= 0 );
921 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
922 lck->lk.depth_locked += 1;
923 return KMP_LOCK_ACQUIRED_NEXT;
926 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
928 lck->lk.depth_locked = 1;
930 lck->lk.owner_id = gtid + 1;
931 return KMP_LOCK_ACQUIRED_FIRST;
936 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
938 char const *
const func =
"omp_set_nest_lock";
939 if ( lck->lk.initialized != lck ) {
940 KMP_FATAL( LockIsUninitialized, func );
942 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
943 KMP_FATAL( LockSimpleUsedAsNestable, func );
945 return __kmp_acquire_nested_ticket_lock( lck, gtid );
949 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
953 KMP_DEBUG_ASSERT( gtid >= 0 );
955 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
956 retval = ++lck->lk.depth_locked;
958 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
963 retval = lck->lk.depth_locked = 1;
965 lck->lk.owner_id = gtid + 1;
971 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
974 char const *
const func =
"omp_test_nest_lock";
975 if ( lck->lk.initialized != lck ) {
976 KMP_FATAL( LockIsUninitialized, func );
978 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
979 KMP_FATAL( LockSimpleUsedAsNestable, func );
981 return __kmp_test_nested_ticket_lock( lck, gtid );
985 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
987 KMP_DEBUG_ASSERT( gtid >= 0 );
990 if ( --(lck->lk.depth_locked) == 0 ) {
992 lck->lk.owner_id = 0;
993 __kmp_release_ticket_lock( lck, gtid );
994 return KMP_LOCK_RELEASED;
996 return KMP_LOCK_STILL_HELD;
1000 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1002 char const *
const func =
"omp_unset_nest_lock";
1004 if ( lck->lk.initialized != lck ) {
1005 KMP_FATAL( LockIsUninitialized, func );
1007 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1008 KMP_FATAL( LockSimpleUsedAsNestable, func );
1010 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1011 KMP_FATAL( LockUnsettingFree, func );
1013 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1014 KMP_FATAL( LockUnsettingSetByAnother, func );
1016 return __kmp_release_nested_ticket_lock( lck, gtid );
1020 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1022 __kmp_init_ticket_lock( lck );
1023 lck->lk.depth_locked = 0;
1027 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1029 __kmp_init_nested_ticket_lock( lck );
1033 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1035 __kmp_destroy_ticket_lock( lck );
1036 lck->lk.depth_locked = 0;
1040 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1042 char const *
const func =
"omp_destroy_nest_lock";
1043 if ( lck->lk.initialized != lck ) {
1044 KMP_FATAL( LockIsUninitialized, func );
1046 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1047 KMP_FATAL( LockSimpleUsedAsNestable, func );
1049 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1050 KMP_FATAL( LockStillOwned, func );
1052 __kmp_destroy_nested_ticket_lock( lck );
1061 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1063 return lck == lck->lk.initialized;
1067 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1069 return lck->lk.location;
1073 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1075 lck->lk.location = loc;
1078 static kmp_lock_flags_t
1079 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1081 return lck->lk.flags;
1085 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1087 lck->lk.flags = flags;
1146 #ifdef DEBUG_QUEUING_LOCKS 1149 #define TRACE_BUF_ELE 1024 1150 static char traces[TRACE_BUF_ELE][128] = { 0 }
1152 #define TRACE_LOCK(X,Y) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y ); 1153 #define TRACE_LOCK_T(X,Y,Z) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X,Y,Z ); 1154 #define TRACE_LOCK_HT(X,Y,Z,Q) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, Z, Q ); 1157 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1158 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1162 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1164 i = tc % TRACE_BUF_ELE;
1165 __kmp_printf_no_lock(
"%s\n", traces[i] );
1166 i = (i+1) % TRACE_BUF_ELE;
1167 while ( i != (tc % TRACE_BUF_ELE) ) {
1168 __kmp_printf_no_lock(
"%s", traces[i] );
1169 i = (i+1) % TRACE_BUF_ELE;
1171 __kmp_printf_no_lock(
"\n" );
1173 __kmp_printf_no_lock(
1174 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1175 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1178 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1180 if ( lck->lk.head_id >= 1 ) {
1181 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1183 __kmp_printf_no_lock(
"-> %d ", t );
1184 t = __kmp_threads[t-1]->th.th_next_waiting;
1187 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1188 __kmp_printf_no_lock(
"\n\n" );
1194 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1196 return TCR_4( lck->lk.owner_id ) - 1;
1200 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1202 return lck->lk.depth_locked != -1;
1206 template <
bool takeTime>
1209 __forceinline
static int 1210 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1213 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1214 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1215 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1216 volatile kmp_uint32 *spin_here_p;
1217 kmp_int32 need_mf = 1;
1220 ompt_state_t prev_state = ompt_state_undefined;
1223 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1225 KMP_FSYNC_PREPARE( lck );
1226 KMP_DEBUG_ASSERT( this_thr != NULL );
1227 spin_here_p = & this_thr->th.th_spin_here;
1229 #ifdef DEBUG_QUEUING_LOCKS 1230 TRACE_LOCK( gtid+1,
"acq ent" );
1232 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1233 if ( this_thr->th.th_next_waiting != 0 )
1234 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1236 KMP_DEBUG_ASSERT( !*spin_here_p );
1237 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1247 *spin_here_p = TRUE;
1260 #ifdef DEBUG_QUEUING_LOCKS 1262 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1271 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1272 KMP_PACK_64( -1, 0 ),
1273 KMP_PACK_64( gtid+1, gtid+1 ) );
1274 #ifdef DEBUG_QUEUING_LOCKS 1275 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1283 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1285 #ifdef DEBUG_QUEUING_LOCKS 1286 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1295 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1297 #ifdef DEBUG_QUEUING_LOCKS 1298 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1306 kmp_int32 grabbed_lock;
1308 #ifdef DEBUG_QUEUING_LOCKS 1310 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1315 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1317 if ( grabbed_lock ) {
1319 *spin_here_p = FALSE;
1321 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1323 #ifdef DEBUG_QUEUING_LOCKS 1324 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1328 if (ompt_enabled && prev_state != ompt_state_undefined) {
1330 this_thr->th.ompt_thread_info.state = prev_state;
1331 this_thr->th.ompt_thread_info.wait_id = 0;
1335 KMP_FSYNC_ACQUIRED( lck );
1336 return KMP_LOCK_ACQUIRED_FIRST;
1344 if (ompt_enabled && prev_state == ompt_state_undefined) {
1346 prev_state = this_thr->th.ompt_thread_info.state;
1347 this_thr->th.ompt_thread_info.wait_id = (uint64_t) lck;
1348 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1354 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1355 KMP_ASSERT( tail_thr != NULL );
1356 tail_thr->th.th_next_waiting = gtid+1;
1359 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1366 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1368 #ifdef DEBUG_QUEUING_LOCKS 1369 TRACE_LOCK( gtid+1,
"acq spin" );
1371 if ( this_thr->th.th_next_waiting != 0 )
1372 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1374 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1375 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1378 #ifdef DEBUG_QUEUING_LOCKS 1379 TRACE_LOCK( gtid+1,
"acq exit 2" );
1384 this_thr->th.ompt_thread_info.state = prev_state;
1385 this_thr->th.ompt_thread_info.wait_id = 0;
1389 return KMP_LOCK_ACQUIRED_FIRST;
1395 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1397 #ifdef DEBUG_QUEUING_LOCKS 1398 TRACE_LOCK( gtid+1,
"acq retry" );
1402 KMP_ASSERT2( 0,
"should not get here" );
1403 return KMP_LOCK_ACQUIRED_FIRST;
1407 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1409 KMP_DEBUG_ASSERT( gtid >= 0 );
1411 return __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1415 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1418 char const *
const func =
"omp_set_lock";
1419 if ( lck->lk.initialized != lck ) {
1420 KMP_FATAL( LockIsUninitialized, func );
1422 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1423 KMP_FATAL( LockNestableUsedAsSimple, func );
1425 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1426 KMP_FATAL( LockIsAlreadyOwned, func );
1429 __kmp_acquire_queuing_lock( lck, gtid );
1431 lck->lk.owner_id = gtid + 1;
1432 return KMP_LOCK_ACQUIRED_FIRST;
1436 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1438 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1441 kmp_info_t *this_thr;
1444 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1445 KMP_DEBUG_ASSERT( gtid >= 0 );
1447 this_thr = __kmp_thread_from_gtid( gtid );
1448 KMP_DEBUG_ASSERT( this_thr != NULL );
1449 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1458 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1459 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1460 KMP_FSYNC_ACQUIRED(lck);
1465 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1470 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1472 char const *
const func =
"omp_test_lock";
1473 if ( lck->lk.initialized != lck ) {
1474 KMP_FATAL( LockIsUninitialized, func );
1476 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1477 KMP_FATAL( LockNestableUsedAsSimple, func );
1480 int retval = __kmp_test_queuing_lock( lck, gtid );
1483 lck->lk.owner_id = gtid + 1;
1489 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1491 register kmp_info_t *this_thr;
1492 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1493 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1495 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1496 KMP_DEBUG_ASSERT( gtid >= 0 );
1497 this_thr = __kmp_thread_from_gtid( gtid );
1498 KMP_DEBUG_ASSERT( this_thr != NULL );
1499 #ifdef DEBUG_QUEUING_LOCKS 1500 TRACE_LOCK( gtid+1,
"rel ent" );
1502 if ( this_thr->th.th_spin_here )
1503 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1504 if ( this_thr->th.th_next_waiting != 0 )
1505 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1507 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1508 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1510 KMP_FSYNC_RELEASING(lck);
1519 #ifdef DEBUG_QUEUING_LOCKS 1521 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1522 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1524 KMP_DEBUG_ASSERT( head != 0 );
1529 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1530 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1532 #ifdef DEBUG_QUEUING_LOCKS 1533 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1540 return KMP_LOCK_RELEASED;
1548 if ( head == tail ) {
1550 #ifdef DEBUG_QUEUING_LOCKS 1551 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1553 KMP_DEBUG_ASSERT( head > 0 );
1556 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1557 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1558 #ifdef DEBUG_QUEUING_LOCKS 1559 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1564 volatile kmp_int32 *waiting_id_p;
1565 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1566 KMP_DEBUG_ASSERT( head_thr != NULL );
1567 waiting_id_p = & head_thr->th.th_next_waiting;
1570 #ifdef DEBUG_QUEUING_LOCKS 1571 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1573 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1579 *head_id_p = KMP_WAIT_YIELD((
volatile kmp_uint32*)waiting_id_p, 0, KMP_NEQ, NULL);
1580 #ifdef DEBUG_QUEUING_LOCKS 1581 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1588 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1589 KMP_DEBUG_ASSERT( head_thr != NULL );
1592 #ifdef DEBUG_QUEUING_LOCKS 1593 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1595 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1600 head_thr->th.th_next_waiting = 0;
1601 #ifdef DEBUG_QUEUING_LOCKS 1602 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1607 head_thr->th.th_spin_here = FALSE;
1609 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1611 #ifdef DEBUG_QUEUING_LOCKS 1612 TRACE_LOCK( gtid+1,
"rel exit 2" );
1614 return KMP_LOCK_RELEASED;
1618 #ifdef DEBUG_QUEUING_LOCKS 1619 TRACE_LOCK( gtid+1,
"rel retry" );
1623 KMP_ASSERT2( 0,
"should not get here" );
1624 return KMP_LOCK_RELEASED;
1628 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1631 char const *
const func =
"omp_unset_lock";
1633 if ( lck->lk.initialized != lck ) {
1634 KMP_FATAL( LockIsUninitialized, func );
1636 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1637 KMP_FATAL( LockNestableUsedAsSimple, func );
1639 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1640 KMP_FATAL( LockUnsettingFree, func );
1642 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1643 KMP_FATAL( LockUnsettingSetByAnother, func );
1645 lck->lk.owner_id = 0;
1646 return __kmp_release_queuing_lock( lck, gtid );
1650 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1652 lck->lk.location = NULL;
1653 lck->lk.head_id = 0;
1654 lck->lk.tail_id = 0;
1655 lck->lk.next_ticket = 0;
1656 lck->lk.now_serving = 0;
1657 lck->lk.owner_id = 0;
1658 lck->lk.depth_locked = -1;
1659 lck->lk.initialized = lck;
1661 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1665 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1667 __kmp_init_queuing_lock( lck );
1671 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1673 lck->lk.initialized = NULL;
1674 lck->lk.location = NULL;
1675 lck->lk.head_id = 0;
1676 lck->lk.tail_id = 0;
1677 lck->lk.next_ticket = 0;
1678 lck->lk.now_serving = 0;
1679 lck->lk.owner_id = 0;
1680 lck->lk.depth_locked = -1;
1684 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1686 char const *
const func =
"omp_destroy_lock";
1687 if ( lck->lk.initialized != lck ) {
1688 KMP_FATAL( LockIsUninitialized, func );
1690 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1691 KMP_FATAL( LockNestableUsedAsSimple, func );
1693 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1694 KMP_FATAL( LockStillOwned, func );
1696 __kmp_destroy_queuing_lock( lck );
1705 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1707 KMP_DEBUG_ASSERT( gtid >= 0 );
1709 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1710 lck->lk.depth_locked += 1;
1711 return KMP_LOCK_ACQUIRED_NEXT;
1714 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1716 lck->lk.depth_locked = 1;
1718 lck->lk.owner_id = gtid + 1;
1719 return KMP_LOCK_ACQUIRED_FIRST;
1724 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1726 char const *
const func =
"omp_set_nest_lock";
1727 if ( lck->lk.initialized != lck ) {
1728 KMP_FATAL( LockIsUninitialized, func );
1730 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1731 KMP_FATAL( LockSimpleUsedAsNestable, func );
1733 return __kmp_acquire_nested_queuing_lock( lck, gtid );
1737 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1741 KMP_DEBUG_ASSERT( gtid >= 0 );
1743 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1744 retval = ++lck->lk.depth_locked;
1746 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1751 retval = lck->lk.depth_locked = 1;
1753 lck->lk.owner_id = gtid + 1;
1759 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1762 char const *
const func =
"omp_test_nest_lock";
1763 if ( lck->lk.initialized != lck ) {
1764 KMP_FATAL( LockIsUninitialized, func );
1766 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1767 KMP_FATAL( LockSimpleUsedAsNestable, func );
1769 return __kmp_test_nested_queuing_lock( lck, gtid );
1773 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1775 KMP_DEBUG_ASSERT( gtid >= 0 );
1778 if ( --(lck->lk.depth_locked) == 0 ) {
1780 lck->lk.owner_id = 0;
1781 __kmp_release_queuing_lock( lck, gtid );
1782 return KMP_LOCK_RELEASED;
1784 return KMP_LOCK_STILL_HELD;
1788 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1790 char const *
const func =
"omp_unset_nest_lock";
1792 if ( lck->lk.initialized != lck ) {
1793 KMP_FATAL( LockIsUninitialized, func );
1795 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1796 KMP_FATAL( LockSimpleUsedAsNestable, func );
1798 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1799 KMP_FATAL( LockUnsettingFree, func );
1801 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1802 KMP_FATAL( LockUnsettingSetByAnother, func );
1804 return __kmp_release_nested_queuing_lock( lck, gtid );
1808 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1810 __kmp_init_queuing_lock( lck );
1811 lck->lk.depth_locked = 0;
1815 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1817 __kmp_init_nested_queuing_lock( lck );
1821 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1823 __kmp_destroy_queuing_lock( lck );
1824 lck->lk.depth_locked = 0;
1828 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1830 char const *
const func =
"omp_destroy_nest_lock";
1831 if ( lck->lk.initialized != lck ) {
1832 KMP_FATAL( LockIsUninitialized, func );
1834 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1835 KMP_FATAL( LockSimpleUsedAsNestable, func );
1837 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1838 KMP_FATAL( LockStillOwned, func );
1840 __kmp_destroy_nested_queuing_lock( lck );
1849 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1851 return lck == lck->lk.initialized;
1855 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1857 return lck->lk.location;
1861 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1863 lck->lk.location = loc;
1866 static kmp_lock_flags_t
1867 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1869 return lck->lk.flags;
1873 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1875 lck->lk.flags = flags;
1878 #if KMP_USE_ADAPTIVE_LOCKS 1884 #if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1886 #include <immintrin.h> 1887 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1892 #define _XBEGIN_STARTED (~0u) 1893 #define _XABORT_EXPLICIT (1 << 0) 1894 #define _XABORT_RETRY (1 << 1) 1895 #define _XABORT_CONFLICT (1 << 2) 1896 #define _XABORT_CAPACITY (1 << 3) 1897 #define _XABORT_DEBUG (1 << 4) 1898 #define _XABORT_NESTED (1 << 5) 1899 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF)) 1902 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1904 #define STRINGIZE_INTERNAL(arg) #arg 1905 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg) 1913 static __inline
int _xbegin()
1942 #endif // KMP_ARCH_X86_64 1952 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n" 1955 "1: movl %%eax,%0\n" 1957 :
"+r"(res)::
"memory",
"%eax");
1958 #endif // KMP_OS_WINDOWS 1965 static __inline
void _xend()
1974 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1984 #define _xabort(ARG) \ 1989 #define _xabort(ARG) \ 1990 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory"); 1993 #endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1998 #if KMP_DEBUG_ADAPTIVE_LOCKS 2003 static kmp_adaptive_lock_statistics_t destroyedStats;
2006 static kmp_adaptive_lock_info_t liveLocks;
2009 static kmp_bootstrap_lock_t chain_lock;
2013 __kmp_init_speculative_stats()
2015 kmp_adaptive_lock_info_t *lck = &liveLocks;
2017 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
2018 lck->stats.next = lck;
2019 lck->stats.prev = lck;
2021 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2022 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2024 __kmp_init_bootstrap_lock( &chain_lock );
2030 __kmp_remember_lock( kmp_adaptive_lock_info_t * lck )
2032 __kmp_acquire_bootstrap_lock( &chain_lock );
2034 lck->stats.next = liveLocks.stats.next;
2035 lck->stats.prev = &liveLocks;
2037 liveLocks.stats.next = lck;
2038 lck->stats.next->stats.prev = lck;
2040 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2041 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2043 __kmp_release_bootstrap_lock( &chain_lock );
2047 __kmp_forget_lock( kmp_adaptive_lock_info_t * lck )
2049 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2050 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2052 kmp_adaptive_lock_info_t * n = lck->stats.next;
2053 kmp_adaptive_lock_info_t * p = lck->stats.prev;
2060 __kmp_zero_speculative_stats( kmp_adaptive_lock_info_t * lck )
2062 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2063 __kmp_remember_lock( lck );
2067 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_info_t * lck )
2069 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2071 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2072 t->successfulSpeculations += s->successfulSpeculations;
2073 t->hardFailedSpeculations += s->hardFailedSpeculations;
2074 t->softFailedSpeculations += s->softFailedSpeculations;
2075 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2076 t->lemmingYields += s->lemmingYields;
2080 __kmp_accumulate_speculative_stats( kmp_adaptive_lock_info_t * lck)
2082 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2084 __kmp_acquire_bootstrap_lock( &chain_lock );
2086 __kmp_add_stats( &destroyedStats, lck );
2087 __kmp_forget_lock( lck );
2089 __kmp_release_bootstrap_lock( &chain_lock );
2093 percent (kmp_uint32 count, kmp_uint32 total)
2095 return (total == 0) ? 0.0: (100.0 * count)/total;
2099 FILE * __kmp_open_stats_file()
2101 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2104 size_t buffLen = KMP_STRLEN( __kmp_speculative_statsfile ) + 20;
2105 char buffer[buffLen];
2106 KMP_SNPRINTF (&buffer[0], buffLen, __kmp_speculative_statsfile,
2107 (kmp_int32)getpid());
2108 FILE * result = fopen(&buffer[0],
"w");
2111 return result ? result : stdout;
2115 __kmp_print_speculative_stats()
2117 if (__kmp_user_lock_kind != lk_adaptive)
2120 FILE * statsFile = __kmp_open_stats_file();
2122 kmp_adaptive_lock_statistics_t total = destroyedStats;
2123 kmp_adaptive_lock_info_t *lck;
2125 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2126 __kmp_add_stats( &total, lck );
2128 kmp_adaptive_lock_statistics_t *t = &total;
2129 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2130 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2131 t->softFailedSpeculations;
2133 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2134 fprintf ( statsFile,
" Lock parameters: \n" 2135 " max_soft_retries : %10d\n" 2136 " max_badness : %10d\n",
2137 __kmp_adaptive_backoff_params.max_soft_retries,
2138 __kmp_adaptive_backoff_params.max_badness);
2139 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2140 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2141 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2142 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2143 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2144 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2145 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2147 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2148 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2149 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2150 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2151 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2152 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2153 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2155 if (statsFile != stdout)
2156 fclose( statsFile );
2159 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ ) 2161 # define KMP_INC_STAT(lck,stat) 2163 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 2166 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2170 bool res = lck->lk.head_id == 0;
2174 #if KMP_COMPILER_ICC 2177 __sync_synchronize();
2184 static __inline
void 2185 __kmp_update_badness_after_success( kmp_adaptive_lock_t *lck )
2188 lck->lk.adaptive.badness = 0;
2189 KMP_INC_STAT(lck,successfulSpeculations);
2193 static __inline
void 2194 __kmp_step_badness( kmp_adaptive_lock_t *lck )
2196 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2197 if ( newBadness > lck->lk.adaptive.max_badness) {
2200 lck->lk.adaptive.badness = newBadness;
2206 __kmp_should_speculate( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2208 kmp_uint32 badness = lck->lk.adaptive.badness;
2209 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2210 int res = (attempts & badness) == 0;
2218 __kmp_test_adaptive_lock_only( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2220 int retries = lck->lk.adaptive.max_soft_retries;
2228 kmp_uint32 status = _xbegin();
2233 if (status == _XBEGIN_STARTED )
2239 if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2245 KMP_ASSERT2( 0,
"should not get here" );
2250 if ( status & SOFT_ABORT_MASK)
2252 KMP_INC_STAT(lck,softFailedSpeculations);
2257 KMP_INC_STAT(lck,hardFailedSpeculations);
2262 }
while( retries-- );
2266 __kmp_step_badness( lck );
2274 __kmp_test_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2277 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2282 lck->lk.adaptive.acquire_attempts++;
2285 if ( __kmp_test_queuing_lock( GET_QLK_PTR(lck), gtid ) )
2287 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2297 __kmp_test_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2299 char const *
const func =
"omp_test_lock";
2300 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2301 KMP_FATAL( LockIsUninitialized, func );
2304 int retval = __kmp_test_adaptive_lock( lck, gtid );
2307 lck->lk.qlk.owner_id = gtid + 1;
2327 __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2329 if ( __kmp_should_speculate( lck, gtid ) )
2331 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2333 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2345 while ( ! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2347 KMP_INC_STAT(lck,lemmingYields);
2351 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2358 lck->lk.adaptive.acquire_attempts++;
2360 __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
2362 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2366 __kmp_acquire_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2368 char const *
const func =
"omp_set_lock";
2369 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2370 KMP_FATAL( LockIsUninitialized, func );
2372 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == gtid ) {
2373 KMP_FATAL( LockIsAlreadyOwned, func );
2376 __kmp_acquire_adaptive_lock( lck, gtid );
2378 lck->lk.qlk.owner_id = gtid + 1;
2382 __kmp_release_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2384 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2389 __kmp_update_badness_after_success( lck );
2394 __kmp_release_queuing_lock( GET_QLK_PTR(lck), gtid );
2396 return KMP_LOCK_RELEASED;
2400 __kmp_release_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2402 char const *
const func =
"omp_unset_lock";
2404 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2405 KMP_FATAL( LockIsUninitialized, func );
2407 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == -1 ) {
2408 KMP_FATAL( LockUnsettingFree, func );
2410 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != gtid ) {
2411 KMP_FATAL( LockUnsettingSetByAnother, func );
2413 lck->lk.qlk.owner_id = 0;
2414 __kmp_release_adaptive_lock( lck, gtid );
2415 return KMP_LOCK_RELEASED;
2419 __kmp_init_adaptive_lock( kmp_adaptive_lock_t *lck )
2421 __kmp_init_queuing_lock( GET_QLK_PTR(lck) );
2422 lck->lk.adaptive.badness = 0;
2423 lck->lk.adaptive.acquire_attempts = 0;
2424 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2425 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2426 #if KMP_DEBUG_ADAPTIVE_LOCKS 2427 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2429 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2433 __kmp_init_adaptive_lock_with_checks( kmp_adaptive_lock_t * lck )
2435 __kmp_init_adaptive_lock( lck );
2439 __kmp_destroy_adaptive_lock( kmp_adaptive_lock_t *lck )
2441 #if KMP_DEBUG_ADAPTIVE_LOCKS 2442 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2444 __kmp_destroy_queuing_lock (GET_QLK_PTR(lck));
2449 __kmp_destroy_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck )
2451 char const *
const func =
"omp_destroy_lock";
2452 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2453 KMP_FATAL( LockIsUninitialized, func );
2455 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != -1 ) {
2456 KMP_FATAL( LockStillOwned, func );
2458 __kmp_destroy_adaptive_lock( lck );
2462 #endif // KMP_USE_ADAPTIVE_LOCKS 2470 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2472 return TCR_4( lck->lk.owner_id ) - 1;
2476 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2478 return lck->lk.depth_locked != -1;
2481 __forceinline
static int 2482 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2484 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2485 kmp_uint64 mask = TCR_8(lck->lk.mask);
2486 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2487 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2488 TCR_PTR(lck->lk.polls);
2490 #ifdef USE_LOCK_PROFILE 2491 if (TCR_8(polls[ticket & mask].poll) != ticket)
2492 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2508 KMP_FSYNC_PREPARE(lck);
2509 KMP_INIT_YIELD(spins);
2510 while (TCR_8(polls[ticket & mask]).poll < ticket) {
2515 KMP_YIELD(TCR_4(__kmp_nth)
2516 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2517 KMP_YIELD_SPIN(spins);
2528 mask = TCR_8(lck->lk.mask);
2529 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2530 TCR_PTR(lck->lk.polls);
2536 KMP_FSYNC_ACQUIRED(lck);
2537 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2539 lck->lk.now_serving = ticket;
2548 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2549 __kmp_free((
void *)lck->lk.old_polls);
2550 lck->lk.old_polls = NULL;
2551 lck->lk.cleanup_ticket = 0;
2559 if (lck->lk.old_polls == NULL) {
2560 bool reconfigure =
false;
2561 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2562 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2564 if (TCR_4(__kmp_nth)
2565 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2570 if (num_polls > 1) {
2572 num_polls = TCR_4(lck->lk.num_polls);
2575 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2576 __kmp_allocate(num_polls *
sizeof(*polls));
2577 polls[0].poll = ticket;
2586 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2587 if (num_waiting > num_polls) {
2588 kmp_uint32 old_num_polls = num_polls;
2591 mask = (mask << 1) | 1;
2593 }
while (num_polls <= num_waiting);
2601 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2602 __kmp_allocate(num_polls *
sizeof(*polls));
2604 for (i = 0; i < old_num_polls; i++) {
2605 polls[i].poll = old_polls[i].poll;
2622 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2623 ticket, lck, num_polls));
2625 lck->lk.old_polls = old_polls;
2626 lck->lk.polls = polls;
2630 lck->lk.num_polls = num_polls;
2631 lck->lk.mask = mask;
2641 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2644 return KMP_LOCK_ACQUIRED_FIRST;
2648 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2650 return __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2654 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2656 char const *
const func =
"omp_set_lock";
2657 if ( lck->lk.initialized != lck ) {
2658 KMP_FATAL( LockIsUninitialized, func );
2660 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2661 KMP_FATAL( LockNestableUsedAsSimple, func );
2663 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2664 KMP_FATAL( LockIsAlreadyOwned, func );
2667 __kmp_acquire_drdpa_lock( lck, gtid );
2669 lck->lk.owner_id = gtid + 1;
2670 return KMP_LOCK_ACQUIRED_FIRST;
2674 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2680 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2681 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2682 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2683 TCR_PTR(lck->lk.polls);
2684 kmp_uint64 mask = TCR_8(lck->lk.mask);
2685 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2686 kmp_uint64 next_ticket = ticket + 1;
2687 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2688 ticket, next_ticket)) {
2689 KMP_FSYNC_ACQUIRED(lck);
2690 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2692 lck->lk.now_serving = ticket;
2710 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2712 char const *
const func =
"omp_test_lock";
2713 if ( lck->lk.initialized != lck ) {
2714 KMP_FATAL( LockIsUninitialized, func );
2716 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2717 KMP_FATAL( LockNestableUsedAsSimple, func );
2720 int retval = __kmp_test_drdpa_lock( lck, gtid );
2723 lck->lk.owner_id = gtid + 1;
2729 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2736 kmp_uint64 ticket = lck->lk.now_serving + 1;
2737 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2738 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2739 TCR_PTR(lck->lk.polls);
2740 kmp_uint64 mask = TCR_8(lck->lk.mask);
2741 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2743 KMP_FSYNC_RELEASING(lck);
2744 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2745 return KMP_LOCK_RELEASED;
2749 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2751 char const *
const func =
"omp_unset_lock";
2753 if ( lck->lk.initialized != lck ) {
2754 KMP_FATAL( LockIsUninitialized, func );
2756 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2757 KMP_FATAL( LockNestableUsedAsSimple, func );
2759 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2760 KMP_FATAL( LockUnsettingFree, func );
2762 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2763 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2764 KMP_FATAL( LockUnsettingSetByAnother, func );
2766 lck->lk.owner_id = 0;
2767 return __kmp_release_drdpa_lock( lck, gtid );
2771 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2773 lck->lk.location = NULL;
2775 lck->lk.num_polls = 1;
2776 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2777 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2778 lck->lk.cleanup_ticket = 0;
2779 lck->lk.old_polls = NULL;
2780 lck->lk.next_ticket = 0;
2781 lck->lk.now_serving = 0;
2782 lck->lk.owner_id = 0;
2783 lck->lk.depth_locked = -1;
2784 lck->lk.initialized = lck;
2786 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2790 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2792 __kmp_init_drdpa_lock( lck );
2796 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2798 lck->lk.initialized = NULL;
2799 lck->lk.location = NULL;
2800 if (lck->lk.polls != NULL) {
2801 __kmp_free((
void *)lck->lk.polls);
2802 lck->lk.polls = NULL;
2804 if (lck->lk.old_polls != NULL) {
2805 __kmp_free((
void *)lck->lk.old_polls);
2806 lck->lk.old_polls = NULL;
2809 lck->lk.num_polls = 0;
2810 lck->lk.cleanup_ticket = 0;
2811 lck->lk.next_ticket = 0;
2812 lck->lk.now_serving = 0;
2813 lck->lk.owner_id = 0;
2814 lck->lk.depth_locked = -1;
2818 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2820 char const *
const func =
"omp_destroy_lock";
2821 if ( lck->lk.initialized != lck ) {
2822 KMP_FATAL( LockIsUninitialized, func );
2824 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2825 KMP_FATAL( LockNestableUsedAsSimple, func );
2827 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2828 KMP_FATAL( LockStillOwned, func );
2830 __kmp_destroy_drdpa_lock( lck );
2839 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2841 KMP_DEBUG_ASSERT( gtid >= 0 );
2843 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2844 lck->lk.depth_locked += 1;
2845 return KMP_LOCK_ACQUIRED_NEXT;
2848 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2850 lck->lk.depth_locked = 1;
2852 lck->lk.owner_id = gtid + 1;
2853 return KMP_LOCK_ACQUIRED_FIRST;
2858 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2860 char const *
const func =
"omp_set_nest_lock";
2861 if ( lck->lk.initialized != lck ) {
2862 KMP_FATAL( LockIsUninitialized, func );
2864 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2865 KMP_FATAL( LockSimpleUsedAsNestable, func );
2867 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2871 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2875 KMP_DEBUG_ASSERT( gtid >= 0 );
2877 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2878 retval = ++lck->lk.depth_locked;
2880 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2885 retval = lck->lk.depth_locked = 1;
2887 lck->lk.owner_id = gtid + 1;
2893 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2895 char const *
const func =
"omp_test_nest_lock";
2896 if ( lck->lk.initialized != lck ) {
2897 KMP_FATAL( LockIsUninitialized, func );
2899 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2900 KMP_FATAL( LockSimpleUsedAsNestable, func );
2902 return __kmp_test_nested_drdpa_lock( lck, gtid );
2906 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2908 KMP_DEBUG_ASSERT( gtid >= 0 );
2911 if ( --(lck->lk.depth_locked) == 0 ) {
2913 lck->lk.owner_id = 0;
2914 __kmp_release_drdpa_lock( lck, gtid );
2915 return KMP_LOCK_RELEASED;
2917 return KMP_LOCK_STILL_HELD;
2921 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2923 char const *
const func =
"omp_unset_nest_lock";
2925 if ( lck->lk.initialized != lck ) {
2926 KMP_FATAL( LockIsUninitialized, func );
2928 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2929 KMP_FATAL( LockSimpleUsedAsNestable, func );
2931 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2932 KMP_FATAL( LockUnsettingFree, func );
2934 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
2935 KMP_FATAL( LockUnsettingSetByAnother, func );
2937 return __kmp_release_nested_drdpa_lock( lck, gtid );
2941 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
2943 __kmp_init_drdpa_lock( lck );
2944 lck->lk.depth_locked = 0;
2948 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2950 __kmp_init_nested_drdpa_lock( lck );
2954 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
2956 __kmp_destroy_drdpa_lock( lck );
2957 lck->lk.depth_locked = 0;
2961 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2963 char const *
const func =
"omp_destroy_nest_lock";
2964 if ( lck->lk.initialized != lck ) {
2965 KMP_FATAL( LockIsUninitialized, func );
2967 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2968 KMP_FATAL( LockSimpleUsedAsNestable, func );
2970 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2971 KMP_FATAL( LockStillOwned, func );
2973 __kmp_destroy_nested_drdpa_lock( lck );
2982 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
2984 return lck == lck->lk.initialized;
2988 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
2990 return lck->lk.location;
2994 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
2996 lck->lk.location = loc;
2999 static kmp_lock_flags_t
3000 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
3002 return lck->lk.flags;
3006 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
3008 lck->lk.flags = flags;
3011 #if KMP_USE_DYNAMIC_LOCK 3014 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq)
3016 TCW_4(*lck, KMP_GET_D_TAG(seq));
3017 KA_TRACE(20, (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
3023 #define HLE_ACQUIRE ".byte 0xf2;" 3024 #define HLE_RELEASE ".byte 0xf3;" 3026 static inline kmp_uint32
3027 swap4(kmp_uint32
volatile *p, kmp_uint32 v)
3029 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" 3037 __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
3043 __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3046 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
3049 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
3050 for (
int i = delay; i != 0; --i)
3052 delay = ((delay << 1) | 1) & 7;
3054 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
3059 __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3061 __kmp_acquire_hle_lock(lck, gtid);
3065 __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3067 __asm__
volatile(HLE_RELEASE
"movl %1,%0" 3069 :
"r"(KMP_LOCK_FREE(hle))
3071 return KMP_LOCK_RELEASED;
3075 __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3077 return __kmp_release_hle_lock(lck, gtid);
3081 __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3083 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
3087 __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3089 return __kmp_test_hle_lock(lck, gtid);
3093 __kmp_init_rtm_lock(kmp_queuing_lock_t *lck)
3095 __kmp_init_queuing_lock(lck);
3099 __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck)
3101 __kmp_destroy_queuing_lock(lck);
3105 __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3107 unsigned retries=3, status;
3110 if (status == _XBEGIN_STARTED) {
3111 if (__kmp_is_unlocked_queuing_lock(lck))
3115 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
3117 while (! __kmp_is_unlocked_queuing_lock(lck))
3120 else if (!(status & _XABORT_RETRY))
3122 }
while (retries--);
3125 __kmp_acquire_queuing_lock(lck, gtid);
3129 __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3131 __kmp_acquire_rtm_lock(lck, gtid);
3135 __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3137 if (__kmp_is_unlocked_queuing_lock(lck)) {
3143 __kmp_release_queuing_lock(lck, gtid);
3145 return KMP_LOCK_RELEASED;
3149 __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3151 return __kmp_release_rtm_lock(lck, gtid);
3155 __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3157 unsigned retries=3, status;
3160 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
3163 if (!(status & _XABORT_RETRY))
3165 }
while (retries--);
3167 return (__kmp_is_unlocked_queuing_lock(lck))? 1: 0;
3171 __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3173 return __kmp_test_rtm_lock(lck, gtid);
3176 #endif // KMP_USE_TSX 3179 static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
3180 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
3181 static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3182 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3183 static int __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3184 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3185 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3186 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3194 #define expand(l, op) 0,__kmp_init_direct_lock, 3195 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
3196 = { __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init) };
3200 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock, 3201 void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *)
3202 = { __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy) };
3206 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 3207 static void (*direct_set[])(kmp_dyna_lock_t *, kmp_int32)
3208 = { __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
3210 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 3211 static void (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32)
3212 = { __kmp_set_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
3216 #define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 3217 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32)
3218 = { __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release) };
3219 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32)
3220 = { __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test) };
3222 #define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 3223 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32)
3224 = { __kmp_unset_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, release) };
3225 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32)
3226 = { __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test) };
3230 void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
3231 int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
3232 int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
3237 #define expand(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock, 3238 void (*__kmp_indirect_init[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, init) };
3239 void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, destroy) };
3243 #define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3244 static void (*indirect_set[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
3246 #define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3247 static void (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
3251 #define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3252 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
3253 static int (*indirect_test[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
3255 #define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3256 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
3257 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
3261 void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
3262 int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
3263 int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
3266 kmp_indirect_lock_table_t __kmp_i_lock_table;
3269 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = { 0 };
3272 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *) = { 0 };
3273 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t) = { 0 };
3274 const ident_t * (*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3275 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3278 static kmp_indirect_lock_t * __kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = { 0 };
3284 kmp_indirect_lock_t *
3285 __kmp_allocate_indirect_lock(
void **user_lock, kmp_int32 gtid, kmp_indirect_locktag_t tag)
3287 kmp_indirect_lock_t *lck;
3288 kmp_lock_index_t idx;
3290 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3292 if (__kmp_indirect_lock_pool[tag] != NULL) {
3294 lck = __kmp_indirect_lock_pool[tag];
3295 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3296 idx = lck->lock->pool.index;
3297 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3298 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n", lck));
3300 idx = __kmp_i_lock_table.next;
3302 if (idx == __kmp_i_lock_table.size) {
3304 int row = __kmp_i_lock_table.size/KMP_I_LOCK_CHUNK;
3305 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3306 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(2*row*
sizeof(kmp_indirect_lock_t *));
3307 KMP_MEMCPY(__kmp_i_lock_table.table, old_table, row*
sizeof(kmp_indirect_lock_t *));
3308 __kmp_free(old_table);
3310 for (
int i = row; i < 2*row; ++i)
3311 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)
3312 __kmp_allocate(KMP_I_LOCK_CHUNK*
sizeof(kmp_indirect_lock_t));
3313 __kmp_i_lock_table.size = 2*idx;
3315 __kmp_i_lock_table.next++;
3316 lck = KMP_GET_I_LOCK(idx);
3318 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3319 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3322 __kmp_release_lock(&__kmp_global_lock, gtid);
3326 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3327 *((kmp_lock_index_t *)user_lock) = idx << 1;
3329 *((kmp_indirect_lock_t **)user_lock) = lck;
3336 static __forceinline
3337 kmp_indirect_lock_t *
3338 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func)
3340 if (__kmp_env_consistency_check) {
3341 kmp_indirect_lock_t *lck = NULL;
3342 if (user_lock == NULL) {
3343 KMP_FATAL(LockIsUninitialized, func);
3345 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3346 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3347 if (idx >= __kmp_i_lock_table.size) {
3348 KMP_FATAL(LockIsUninitialized, func);
3350 lck = KMP_GET_I_LOCK(idx);
3352 lck = *((kmp_indirect_lock_t **)user_lock);
3355 KMP_FATAL(LockIsUninitialized, func);
3359 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3360 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3362 return *((kmp_indirect_lock_t **)user_lock);
3368 __kmp_init_indirect_lock(kmp_dyna_lock_t * lock, kmp_dyna_lockseq_t seq)
3370 #if KMP_USE_ADAPTIVE_LOCKS 3371 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3372 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3373 seq = lockseq_queuing;
3377 if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
3378 seq = lockseq_queuing;
3381 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3382 kmp_indirect_lock_t *l = __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3383 KMP_I_LOCK_FUNC(l, init)(l->lock);
3384 KA_TRACE(20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n", seq));
3388 __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock)
3390 kmp_uint32 gtid = __kmp_entry_gtid();
3391 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3392 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3393 kmp_indirect_locktag_t tag = l->type;
3395 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3398 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3399 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3400 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3402 __kmp_indirect_lock_pool[tag] = l;
3404 __kmp_release_lock(&__kmp_global_lock, gtid);
3408 __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3410 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3411 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3415 __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3417 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3418 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3422 __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3424 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3425 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3429 __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3431 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3432 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3436 __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3438 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3439 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3443 __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3445 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3446 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3449 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3453 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
3457 case lockseq_nested_tas:
3458 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3461 case lockseq_nested_futex:
3462 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3464 case lockseq_ticket:
3465 case lockseq_nested_ticket:
3466 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3467 case lockseq_queuing:
3468 case lockseq_nested_queuing:
3469 #if KMP_USE_ADAPTIVE_LOCKS 3470 case lockseq_adaptive:
3471 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3474 case lockseq_nested_drdpa:
3475 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3483 __kmp_init_dynamic_user_locks()
3486 if (__kmp_env_consistency_check) {
3487 __kmp_direct_set = direct_set_check;
3488 __kmp_direct_unset = direct_unset_check;
3489 __kmp_direct_test = direct_test_check;
3490 __kmp_indirect_set = indirect_set_check;
3491 __kmp_indirect_unset = indirect_unset_check;
3492 __kmp_indirect_test = indirect_test_check;
3495 __kmp_direct_set = direct_set;
3496 __kmp_direct_unset = direct_unset;
3497 __kmp_direct_test = direct_test;
3498 __kmp_indirect_set = indirect_set;
3499 __kmp_indirect_unset = indirect_unset;
3500 __kmp_indirect_test = indirect_test;
3504 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3505 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3506 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)
3507 __kmp_allocate(KMP_I_LOCK_CHUNK*
sizeof(kmp_indirect_lock_t));
3508 __kmp_i_lock_table.next = 0;
3511 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3512 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3513 #if KMP_USE_ADAPTIVE_LOCKS 3514 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3516 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3518 __kmp_indirect_lock_size[locktag_rtm] =
sizeof(kmp_queuing_lock_t);
3520 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3522 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3524 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3525 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3526 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3529 #define fill_jumps(table, expand, sep) { \ 3530 table[locktag##sep##ticket] = expand(ticket); \ 3531 table[locktag##sep##queuing] = expand(queuing); \ 3532 table[locktag##sep##drdpa] = expand(drdpa); \ 3535 #if KMP_USE_ADAPTIVE_LOCKS 3536 # define fill_table(table, expand) { \ 3537 fill_jumps(table, expand, _); \ 3538 table[locktag_adaptive] = expand(queuing); \ 3539 fill_jumps(table, expand, _nested_); \ 3542 # define fill_table(table, expand) { \ 3543 fill_jumps(table, expand, _); \ 3544 fill_jumps(table, expand, _nested_); \ 3546 #endif // KMP_USE_ADAPTIVE_LOCKS 3548 #define expand(l) (void (*)(kmp_user_lock_p, const ident_t *))__kmp_set_##l##_lock_location 3549 fill_table(__kmp_indirect_set_location, expand);
3551 #define expand(l) (void (*)(kmp_user_lock_p, kmp_lock_flags_t))__kmp_set_##l##_lock_flags 3552 fill_table(__kmp_indirect_set_flags, expand);
3554 #define expand(l) (const ident_t * (*)(kmp_user_lock_p))__kmp_get_##l##_lock_location 3555 fill_table(__kmp_indirect_get_location, expand);
3557 #define expand(l) (kmp_lock_flags_t (*)(kmp_user_lock_p))__kmp_get_##l##_lock_flags 3558 fill_table(__kmp_indirect_get_flags, expand);
3561 __kmp_init_user_locks = TRUE;
3566 __kmp_cleanup_indirect_user_locks()
3572 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3573 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3575 kmp_indirect_lock_t *ll = l;
3576 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3577 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n", ll));
3578 __kmp_free(ll->lock);
3581 __kmp_indirect_lock_pool[k] = NULL;
3584 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3585 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3586 if (l->lock != NULL) {
3588 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3589 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n", l));
3590 __kmp_free(l->lock);
3594 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3595 __kmp_free(__kmp_i_lock_table.table[i]);
3596 __kmp_free(__kmp_i_lock_table.table);
3598 __kmp_init_user_locks = FALSE;
3601 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3602 int __kmp_num_locks_in_block = 1;
3604 #else // KMP_USE_DYNAMIC_LOCK 3613 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3615 size_t __kmp_base_user_lock_size = 0;
3616 size_t __kmp_user_lock_size = 0;
3618 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
3619 int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3621 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3622 int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3623 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3624 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
3625 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3626 int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3628 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3629 int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3630 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3631 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3633 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
3634 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
3635 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
3636 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
3637 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
3639 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
3641 switch ( user_lock_kind ) {
3647 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
3648 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3650 __kmp_get_user_lock_owner_ =
3651 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3652 ( &__kmp_get_tas_lock_owner );
3654 if ( __kmp_env_consistency_check ) {
3655 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3656 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3659 KMP_BIND_USER_LOCK(tas);
3660 KMP_BIND_NESTED_USER_LOCK(tas);
3663 __kmp_destroy_user_lock_ =
3664 ( void ( * )( kmp_user_lock_p ) )
3665 ( &__kmp_destroy_tas_lock );
3667 __kmp_is_user_lock_initialized_ =
3668 ( int ( * )( kmp_user_lock_p ) ) NULL;
3670 __kmp_get_user_lock_location_ =
3671 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3673 __kmp_set_user_lock_location_ =
3674 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3676 __kmp_get_user_lock_flags_ =
3677 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3679 __kmp_set_user_lock_flags_ =
3680 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3684 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) 3687 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3688 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3690 __kmp_get_user_lock_owner_ =
3691 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3692 ( &__kmp_get_futex_lock_owner );
3694 if ( __kmp_env_consistency_check ) {
3695 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3696 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3699 KMP_BIND_USER_LOCK(futex);
3700 KMP_BIND_NESTED_USER_LOCK(futex);
3703 __kmp_destroy_user_lock_ =
3704 ( void ( * )( kmp_user_lock_p ) )
3705 ( &__kmp_destroy_futex_lock );
3707 __kmp_is_user_lock_initialized_ =
3708 ( int ( * )( kmp_user_lock_p ) ) NULL;
3710 __kmp_get_user_lock_location_ =
3711 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3713 __kmp_set_user_lock_location_ =
3714 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3716 __kmp_get_user_lock_flags_ =
3717 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3719 __kmp_set_user_lock_flags_ =
3720 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3724 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) 3727 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3728 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3730 __kmp_get_user_lock_owner_ =
3731 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3732 ( &__kmp_get_ticket_lock_owner );
3734 if ( __kmp_env_consistency_check ) {
3735 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3736 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3739 KMP_BIND_USER_LOCK(ticket);
3740 KMP_BIND_NESTED_USER_LOCK(ticket);
3743 __kmp_destroy_user_lock_ =
3744 ( void ( * )( kmp_user_lock_p ) )
3745 ( &__kmp_destroy_ticket_lock );
3747 __kmp_is_user_lock_initialized_ =
3748 ( int ( * )( kmp_user_lock_p ) )
3749 ( &__kmp_is_ticket_lock_initialized );
3751 __kmp_get_user_lock_location_ =
3752 (
const ident_t * ( * )( kmp_user_lock_p ) )
3753 ( &__kmp_get_ticket_lock_location );
3755 __kmp_set_user_lock_location_ =
3756 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3757 ( &__kmp_set_ticket_lock_location );
3759 __kmp_get_user_lock_flags_ =
3760 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3761 ( &__kmp_get_ticket_lock_flags );
3763 __kmp_set_user_lock_flags_ =
3764 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3765 ( &__kmp_set_ticket_lock_flags );
3770 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3771 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3773 __kmp_get_user_lock_owner_ =
3774 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3775 ( &__kmp_get_queuing_lock_owner );
3777 if ( __kmp_env_consistency_check ) {
3778 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3779 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3782 KMP_BIND_USER_LOCK(queuing);
3783 KMP_BIND_NESTED_USER_LOCK(queuing);
3786 __kmp_destroy_user_lock_ =
3787 ( void ( * )( kmp_user_lock_p ) )
3788 ( &__kmp_destroy_queuing_lock );
3790 __kmp_is_user_lock_initialized_ =
3791 ( int ( * )( kmp_user_lock_p ) )
3792 ( &__kmp_is_queuing_lock_initialized );
3794 __kmp_get_user_lock_location_ =
3795 (
const ident_t * ( * )( kmp_user_lock_p ) )
3796 ( &__kmp_get_queuing_lock_location );
3798 __kmp_set_user_lock_location_ =
3799 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3800 ( &__kmp_set_queuing_lock_location );
3802 __kmp_get_user_lock_flags_ =
3803 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3804 ( &__kmp_get_queuing_lock_flags );
3806 __kmp_set_user_lock_flags_ =
3807 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3808 ( &__kmp_set_queuing_lock_flags );
3812 #if KMP_USE_ADAPTIVE_LOCKS 3814 __kmp_base_user_lock_size =
sizeof( kmp_base_adaptive_lock_t );
3815 __kmp_user_lock_size =
sizeof( kmp_adaptive_lock_t );
3817 __kmp_get_user_lock_owner_ =
3818 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3819 ( &__kmp_get_queuing_lock_owner );
3821 if ( __kmp_env_consistency_check ) {
3822 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3825 KMP_BIND_USER_LOCK(adaptive);
3828 __kmp_destroy_user_lock_ =
3829 ( void ( * )( kmp_user_lock_p ) )
3830 ( &__kmp_destroy_adaptive_lock );
3832 __kmp_is_user_lock_initialized_ =
3833 ( int ( * )( kmp_user_lock_p ) )
3834 ( &__kmp_is_queuing_lock_initialized );
3836 __kmp_get_user_lock_location_ =
3837 (
const ident_t * ( * )( kmp_user_lock_p ) )
3838 ( &__kmp_get_queuing_lock_location );
3840 __kmp_set_user_lock_location_ =
3841 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3842 ( &__kmp_set_queuing_lock_location );
3844 __kmp_get_user_lock_flags_ =
3845 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3846 ( &__kmp_get_queuing_lock_flags );
3848 __kmp_set_user_lock_flags_ =
3849 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3850 ( &__kmp_set_queuing_lock_flags );
3854 #endif // KMP_USE_ADAPTIVE_LOCKS 3857 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3858 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3860 __kmp_get_user_lock_owner_ =
3861 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3862 ( &__kmp_get_drdpa_lock_owner );
3864 if ( __kmp_env_consistency_check ) {
3865 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3866 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3869 KMP_BIND_USER_LOCK(drdpa);
3870 KMP_BIND_NESTED_USER_LOCK(drdpa);
3873 __kmp_destroy_user_lock_ =
3874 ( void ( * )( kmp_user_lock_p ) )
3875 ( &__kmp_destroy_drdpa_lock );
3877 __kmp_is_user_lock_initialized_ =
3878 ( int ( * )( kmp_user_lock_p ) )
3879 ( &__kmp_is_drdpa_lock_initialized );
3881 __kmp_get_user_lock_location_ =
3882 (
const ident_t * ( * )( kmp_user_lock_p ) )
3883 ( &__kmp_get_drdpa_lock_location );
3885 __kmp_set_user_lock_location_ =
3886 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3887 ( &__kmp_set_drdpa_lock_location );
3889 __kmp_get_user_lock_flags_ =
3890 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3891 ( &__kmp_get_drdpa_lock_flags );
3893 __kmp_set_user_lock_flags_ =
3894 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3895 ( &__kmp_set_drdpa_lock_flags );
3905 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3906 kmp_user_lock_p __kmp_lock_pool = NULL;
3909 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3910 int __kmp_num_locks_in_block = 1;
3912 static kmp_lock_index_t
3913 __kmp_lock_table_insert( kmp_user_lock_p lck )
3916 kmp_lock_index_t index;
3917 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3918 kmp_lock_index_t size;
3919 kmp_user_lock_p *table;
3921 if ( __kmp_user_lock_table.allocated == 0 ) {
3925 size = __kmp_user_lock_table.allocated * 2;
3927 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
3928 KMP_MEMCPY( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
3929 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3934 __kmp_user_lock_table.table = table;
3935 __kmp_user_lock_table.allocated = size;
3937 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
3938 index = __kmp_user_lock_table.used;
3939 __kmp_user_lock_table.table[ index ] = lck;
3940 ++ __kmp_user_lock_table.used;
3944 static kmp_user_lock_p
3945 __kmp_lock_block_allocate()
3948 static int last_index = 0;
3949 if ( ( last_index >= __kmp_num_locks_in_block )
3950 || ( __kmp_lock_blocks == NULL ) ) {
3954 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3955 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3956 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
3958 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
3959 new_block->next_block = __kmp_lock_blocks;
3960 new_block->locks = (
void *)buffer;
3963 __kmp_lock_blocks = new_block;
3965 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
3966 [ last_index * __kmp_user_lock_size ] ) );
3976 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3977 kmp_lock_flags_t flags )
3979 kmp_user_lock_p lck;
3980 kmp_lock_index_t index;
3981 KMP_DEBUG_ASSERT( user_lock );
3983 __kmp_acquire_lock( &__kmp_global_lock, gtid );
3985 if ( __kmp_lock_pool == NULL ) {
3987 if ( __kmp_num_locks_in_block <= 1 ) {
3988 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
3991 lck = __kmp_lock_block_allocate();
3996 index = __kmp_lock_table_insert( lck );
4000 lck = __kmp_lock_pool;
4001 index = __kmp_lock_pool->pool.index;
4002 __kmp_lock_pool = __kmp_lock_pool->pool.next;
4009 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4010 * ( (kmp_lock_index_t *) user_lock ) = index;
4013 * ( (kmp_user_lock_p *) user_lock ) = lck;
4017 __kmp_set_user_lock_flags( lck, flags );
4019 __kmp_release_lock( & __kmp_global_lock, gtid );
4026 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
4028 KMP_DEBUG_ASSERT( user_lock != NULL );
4029 KMP_DEBUG_ASSERT( lck != NULL );
4031 __kmp_acquire_lock( & __kmp_global_lock, gtid );
4033 lck->pool.next = __kmp_lock_pool;
4034 __kmp_lock_pool = lck;
4035 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4036 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
4037 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
4038 lck->pool.index = index;
4041 __kmp_release_lock( & __kmp_global_lock, gtid );
4045 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
4047 kmp_user_lock_p lck = NULL;
4049 if ( __kmp_env_consistency_check ) {
4050 if ( user_lock == NULL ) {
4051 KMP_FATAL( LockIsUninitialized, func );
4055 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4056 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
4057 if ( __kmp_env_consistency_check ) {
4058 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
4059 KMP_FATAL( LockIsUninitialized, func );
4062 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
4063 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
4064 lck = __kmp_user_lock_table.table[index];
4067 lck = *( (kmp_user_lock_p *)user_lock );
4070 if ( __kmp_env_consistency_check ) {
4071 if ( lck == NULL ) {
4072 KMP_FATAL( LockIsUninitialized, func );
4080 __kmp_cleanup_user_locks(
void )
4087 __kmp_lock_pool = NULL;
4089 #define IS_CRITICAL(lck) \ 4090 ( ( __kmp_get_user_lock_flags_ != NULL ) && \ 4091 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) ) 4121 while ( __kmp_user_lock_table.used > 1 ) {
4128 kmp_user_lock_p lck = __kmp_user_lock_table.table[
4129 --__kmp_user_lock_table.used ];
4131 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
4132 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
4139 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
4140 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
4142 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
4143 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.line );
4144 __kmp_str_loc_free( &str_loc);
4148 if ( IS_CRITICAL( lck ) ) {
4149 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
4152 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
4160 __kmp_destroy_user_lock( lck );
4166 if ( __kmp_lock_blocks == NULL ) {
4176 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4177 __kmp_user_lock_table.table = NULL;
4178 __kmp_user_lock_table.allocated = 0;
4180 while ( table_ptr != NULL ) {
4185 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
4186 __kmp_free( table_ptr );
4193 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4194 __kmp_lock_blocks = NULL;
4196 while ( block_ptr != NULL ) {
4197 kmp_block_of_locks_t *next = block_ptr->next_block;
4198 __kmp_free( block_ptr->locks );
4205 TCW_4(__kmp_init_user_locks, FALSE);
4208 #endif // KMP_USE_DYNAMIC_LOCK