22 #include "kmp_wait_release.h" 23 #include "kmp_wrapper_getpid.h" 25 #include "tsan_annotations.h" 28 #include <sys/syscall.h> 46 void __kmp_validate_locks(
void) {
51 x = ~((kmp_uint32)0) - 2;
54 for (i = 0; i < 8; ++i, ++x, ++y) {
55 kmp_uint32 z = (x - y);
59 KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
73 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
74 return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1;
77 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
78 return lck->lk.depth_locked != -1;
81 __forceinline
static int 82 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
85 #ifdef USE_LOCK_PROFILE 86 kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll);
87 if ((curr != 0) && (curr != gtid + 1))
88 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
92 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
93 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
95 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
96 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
97 KMP_FSYNC_ACQUIRED(lck);
98 return KMP_LOCK_ACQUIRED_FIRST;
102 KMP_FSYNC_PREPARE(lck);
103 KMP_INIT_YIELD(spins);
104 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
107 KMP_YIELD_SPIN(spins);
110 kmp_backoff_t backoff = __kmp_spin_backoff_params;
111 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
112 !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
113 __kmp_spin_backoff(&backoff);
114 if (TCR_4(__kmp_nth) >
115 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
118 KMP_YIELD_SPIN(spins);
121 KMP_FSYNC_ACQUIRED(lck);
122 return KMP_LOCK_ACQUIRED_FIRST;
125 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
126 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
127 ANNOTATE_TAS_ACQUIRED(lck);
131 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
133 char const *
const func =
"omp_set_lock";
134 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
135 __kmp_is_tas_lock_nestable(lck)) {
136 KMP_FATAL(LockNestableUsedAsSimple, func);
138 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
139 KMP_FATAL(LockIsAlreadyOwned, func);
141 return __kmp_acquire_tas_lock(lck, gtid);
144 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
145 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
146 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
147 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
148 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
149 KMP_FSYNC_ACQUIRED(lck);
155 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
157 char const *
const func =
"omp_test_lock";
158 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
159 __kmp_is_tas_lock_nestable(lck)) {
160 KMP_FATAL(LockNestableUsedAsSimple, func);
162 return __kmp_test_tas_lock(lck, gtid);
165 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
168 KMP_FSYNC_RELEASING(lck);
169 ANNOTATE_TAS_RELEASED(lck);
170 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas));
173 KMP_YIELD(TCR_4(__kmp_nth) >
174 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
175 return KMP_LOCK_RELEASED;
178 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
180 char const *
const func =
"omp_unset_lock";
182 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
183 __kmp_is_tas_lock_nestable(lck)) {
184 KMP_FATAL(LockNestableUsedAsSimple, func);
186 if (__kmp_get_tas_lock_owner(lck) == -1) {
187 KMP_FATAL(LockUnsettingFree, func);
189 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
190 (__kmp_get_tas_lock_owner(lck) != gtid)) {
191 KMP_FATAL(LockUnsettingSetByAnother, func);
193 return __kmp_release_tas_lock(lck, gtid);
196 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
197 lck->lk.poll = KMP_LOCK_FREE(tas);
200 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
201 __kmp_init_tas_lock(lck);
204 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
206 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
207 char const *
const func =
"omp_destroy_lock";
208 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
209 __kmp_is_tas_lock_nestable(lck)) {
210 KMP_FATAL(LockNestableUsedAsSimple, func);
212 if (__kmp_get_tas_lock_owner(lck) != -1) {
213 KMP_FATAL(LockStillOwned, func);
215 __kmp_destroy_tas_lock(lck);
220 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
221 KMP_DEBUG_ASSERT(gtid >= 0);
223 if (__kmp_get_tas_lock_owner(lck) == gtid) {
224 lck->lk.depth_locked += 1;
225 return KMP_LOCK_ACQUIRED_NEXT;
227 __kmp_acquire_tas_lock_timed_template(lck, gtid);
228 ANNOTATE_TAS_ACQUIRED(lck);
229 lck->lk.depth_locked = 1;
230 return KMP_LOCK_ACQUIRED_FIRST;
234 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
236 char const *
const func =
"omp_set_nest_lock";
237 if (!__kmp_is_tas_lock_nestable(lck)) {
238 KMP_FATAL(LockSimpleUsedAsNestable, func);
240 return __kmp_acquire_nested_tas_lock(lck, gtid);
243 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
246 KMP_DEBUG_ASSERT(gtid >= 0);
248 if (__kmp_get_tas_lock_owner(lck) == gtid) {
249 retval = ++lck->lk.depth_locked;
250 }
else if (!__kmp_test_tas_lock(lck, gtid)) {
254 retval = lck->lk.depth_locked = 1;
259 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
261 char const *
const func =
"omp_test_nest_lock";
262 if (!__kmp_is_tas_lock_nestable(lck)) {
263 KMP_FATAL(LockSimpleUsedAsNestable, func);
265 return __kmp_test_nested_tas_lock(lck, gtid);
268 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
269 KMP_DEBUG_ASSERT(gtid >= 0);
272 if (--(lck->lk.depth_locked) == 0) {
273 __kmp_release_tas_lock(lck, gtid);
274 return KMP_LOCK_RELEASED;
276 return KMP_LOCK_STILL_HELD;
279 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
281 char const *
const func =
"omp_unset_nest_lock";
283 if (!__kmp_is_tas_lock_nestable(lck)) {
284 KMP_FATAL(LockSimpleUsedAsNestable, func);
286 if (__kmp_get_tas_lock_owner(lck) == -1) {
287 KMP_FATAL(LockUnsettingFree, func);
289 if (__kmp_get_tas_lock_owner(lck) != gtid) {
290 KMP_FATAL(LockUnsettingSetByAnother, func);
292 return __kmp_release_nested_tas_lock(lck, gtid);
295 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
296 __kmp_init_tas_lock(lck);
297 lck->lk.depth_locked = 0;
300 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
301 __kmp_init_nested_tas_lock(lck);
304 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
305 __kmp_destroy_tas_lock(lck);
306 lck->lk.depth_locked = 0;
309 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
310 char const *
const func =
"omp_destroy_nest_lock";
311 if (!__kmp_is_tas_lock_nestable(lck)) {
312 KMP_FATAL(LockSimpleUsedAsNestable, func);
314 if (__kmp_get_tas_lock_owner(lck) != -1) {
315 KMP_FATAL(LockStillOwned, func);
317 __kmp_destroy_nested_tas_lock(lck);
330 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
331 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
334 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
335 return lck->lk.depth_locked != -1;
338 __forceinline
static int 339 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
340 kmp_int32 gtid_code = (gtid + 1) << 1;
344 #ifdef USE_LOCK_PROFILE 345 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
346 if ((curr != 0) && (curr != gtid_code))
347 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
351 KMP_FSYNC_PREPARE(lck);
352 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
353 lck, lck->lk.poll, gtid));
357 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(
358 &(lck->lk.poll), KMP_LOCK_FREE(futex),
359 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) {
361 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
364 (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
365 lck, gtid, poll_val, cond));
376 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
377 poll_val | KMP_LOCK_BUSY(1, futex))) {
380 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
381 lck, lck->lk.poll, gtid));
384 poll_val |= KMP_LOCK_BUSY(1, futex);
387 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
388 lck->lk.poll, gtid));
393 (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
394 lck, gtid, poll_val));
397 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
399 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) " 400 "failed (rc=%d errno=%d)\n",
401 lck, gtid, poll_val, rc, errno));
406 (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
407 lck, gtid, poll_val));
414 KMP_FSYNC_ACQUIRED(lck);
415 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
416 lck->lk.poll, gtid));
417 return KMP_LOCK_ACQUIRED_FIRST;
420 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
421 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
422 ANNOTATE_FUTEX_ACQUIRED(lck);
426 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
428 char const *
const func =
"omp_set_lock";
429 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
430 __kmp_is_futex_lock_nestable(lck)) {
431 KMP_FATAL(LockNestableUsedAsSimple, func);
433 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
434 KMP_FATAL(LockIsAlreadyOwned, func);
436 return __kmp_acquire_futex_lock(lck, gtid);
439 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
440 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
441 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) {
442 KMP_FSYNC_ACQUIRED(lck);
448 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
450 char const *
const func =
"omp_test_lock";
451 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
452 __kmp_is_futex_lock_nestable(lck)) {
453 KMP_FATAL(LockNestableUsedAsSimple, func);
455 return __kmp_test_futex_lock(lck, gtid);
458 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
461 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
462 lck, lck->lk.poll, gtid));
464 KMP_FSYNC_RELEASING(lck);
465 ANNOTATE_FUTEX_RELEASED(lck);
467 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
470 (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
471 lck, gtid, poll_val));
473 if (KMP_LOCK_STRIP(poll_val) & 1) {
475 (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
477 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
483 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
484 lck->lk.poll, gtid));
486 KMP_YIELD(TCR_4(__kmp_nth) >
487 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
488 return KMP_LOCK_RELEASED;
491 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
493 char const *
const func =
"omp_unset_lock";
495 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
496 __kmp_is_futex_lock_nestable(lck)) {
497 KMP_FATAL(LockNestableUsedAsSimple, func);
499 if (__kmp_get_futex_lock_owner(lck) == -1) {
500 KMP_FATAL(LockUnsettingFree, func);
502 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
503 (__kmp_get_futex_lock_owner(lck) != gtid)) {
504 KMP_FATAL(LockUnsettingSetByAnother, func);
506 return __kmp_release_futex_lock(lck, gtid);
509 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
510 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
513 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
514 __kmp_init_futex_lock(lck);
517 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
519 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
520 char const *
const func =
"omp_destroy_lock";
521 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
522 __kmp_is_futex_lock_nestable(lck)) {
523 KMP_FATAL(LockNestableUsedAsSimple, func);
525 if (__kmp_get_futex_lock_owner(lck) != -1) {
526 KMP_FATAL(LockStillOwned, func);
528 __kmp_destroy_futex_lock(lck);
533 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
534 KMP_DEBUG_ASSERT(gtid >= 0);
536 if (__kmp_get_futex_lock_owner(lck) == gtid) {
537 lck->lk.depth_locked += 1;
538 return KMP_LOCK_ACQUIRED_NEXT;
540 __kmp_acquire_futex_lock_timed_template(lck, gtid);
541 ANNOTATE_FUTEX_ACQUIRED(lck);
542 lck->lk.depth_locked = 1;
543 return KMP_LOCK_ACQUIRED_FIRST;
547 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
549 char const *
const func =
"omp_set_nest_lock";
550 if (!__kmp_is_futex_lock_nestable(lck)) {
551 KMP_FATAL(LockSimpleUsedAsNestable, func);
553 return __kmp_acquire_nested_futex_lock(lck, gtid);
556 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
559 KMP_DEBUG_ASSERT(gtid >= 0);
561 if (__kmp_get_futex_lock_owner(lck) == gtid) {
562 retval = ++lck->lk.depth_locked;
563 }
else if (!__kmp_test_futex_lock(lck, gtid)) {
567 retval = lck->lk.depth_locked = 1;
572 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
574 char const *
const func =
"omp_test_nest_lock";
575 if (!__kmp_is_futex_lock_nestable(lck)) {
576 KMP_FATAL(LockSimpleUsedAsNestable, func);
578 return __kmp_test_nested_futex_lock(lck, gtid);
581 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
582 KMP_DEBUG_ASSERT(gtid >= 0);
585 if (--(lck->lk.depth_locked) == 0) {
586 __kmp_release_futex_lock(lck, gtid);
587 return KMP_LOCK_RELEASED;
589 return KMP_LOCK_STILL_HELD;
592 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
594 char const *
const func =
"omp_unset_nest_lock";
596 if (!__kmp_is_futex_lock_nestable(lck)) {
597 KMP_FATAL(LockSimpleUsedAsNestable, func);
599 if (__kmp_get_futex_lock_owner(lck) == -1) {
600 KMP_FATAL(LockUnsettingFree, func);
602 if (__kmp_get_futex_lock_owner(lck) != gtid) {
603 KMP_FATAL(LockUnsettingSetByAnother, func);
605 return __kmp_release_nested_futex_lock(lck, gtid);
608 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
609 __kmp_init_futex_lock(lck);
610 lck->lk.depth_locked = 0;
613 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
614 __kmp_init_nested_futex_lock(lck);
617 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
618 __kmp_destroy_futex_lock(lck);
619 lck->lk.depth_locked = 0;
622 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
623 char const *
const func =
"omp_destroy_nest_lock";
624 if (!__kmp_is_futex_lock_nestable(lck)) {
625 KMP_FATAL(LockSimpleUsedAsNestable, func);
627 if (__kmp_get_futex_lock_owner(lck) != -1) {
628 KMP_FATAL(LockStillOwned, func);
630 __kmp_destroy_nested_futex_lock(lck);
633 #endif // KMP_USE_FUTEX 638 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
639 return std::atomic_load_explicit(&lck->lk.owner_id,
640 std::memory_order_relaxed) -
644 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
645 return std::atomic_load_explicit(&lck->lk.depth_locked,
646 std::memory_order_relaxed) != -1;
649 static kmp_uint32 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket) {
650 return std::atomic_load_explicit((std::atomic<unsigned> *)now_serving,
651 std::memory_order_acquire) == my_ticket;
654 __forceinline
static int 655 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
657 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit(
658 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
660 #ifdef USE_LOCK_PROFILE 661 if (std::atomic_load_explicit(&lck->lk.now_serving,
662 std::memory_order_relaxed) != my_ticket)
663 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
667 if (std::atomic_load_explicit(&lck->lk.now_serving,
668 std::memory_order_acquire) == my_ticket) {
669 return KMP_LOCK_ACQUIRED_FIRST;
671 KMP_WAIT_YIELD_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
672 return KMP_LOCK_ACQUIRED_FIRST;
675 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
676 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
677 ANNOTATE_TICKET_ACQUIRED(lck);
681 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
683 char const *
const func =
"omp_set_lock";
685 if (!std::atomic_load_explicit(&lck->lk.initialized,
686 std::memory_order_relaxed)) {
687 KMP_FATAL(LockIsUninitialized, func);
689 if (lck->lk.self != lck) {
690 KMP_FATAL(LockIsUninitialized, func);
692 if (__kmp_is_ticket_lock_nestable(lck)) {
693 KMP_FATAL(LockNestableUsedAsSimple, func);
695 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
696 KMP_FATAL(LockIsAlreadyOwned, func);
699 __kmp_acquire_ticket_lock(lck, gtid);
701 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
702 std::memory_order_relaxed);
703 return KMP_LOCK_ACQUIRED_FIRST;
706 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
707 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
708 std::memory_order_relaxed);
710 if (std::atomic_load_explicit(&lck->lk.now_serving,
711 std::memory_order_relaxed) == my_ticket) {
712 kmp_uint32 next_ticket = my_ticket + 1;
713 if (std::atomic_compare_exchange_strong_explicit(
714 &lck->lk.next_ticket, &my_ticket, next_ticket,
715 std::memory_order_acquire, std::memory_order_acquire)) {
722 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
724 char const *
const func =
"omp_test_lock";
726 if (!std::atomic_load_explicit(&lck->lk.initialized,
727 std::memory_order_relaxed)) {
728 KMP_FATAL(LockIsUninitialized, func);
730 if (lck->lk.self != lck) {
731 KMP_FATAL(LockIsUninitialized, func);
733 if (__kmp_is_ticket_lock_nestable(lck)) {
734 KMP_FATAL(LockNestableUsedAsSimple, func);
737 int retval = __kmp_test_ticket_lock(lck, gtid);
740 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
741 std::memory_order_relaxed);
746 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
747 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
748 std::memory_order_relaxed) -
749 std::atomic_load_explicit(&lck->lk.now_serving,
750 std::memory_order_relaxed);
752 ANNOTATE_TICKET_RELEASED(lck);
753 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
754 std::memory_order_release);
757 (kmp_uint32)(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
758 return KMP_LOCK_RELEASED;
761 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
763 char const *
const func =
"omp_unset_lock";
765 if (!std::atomic_load_explicit(&lck->lk.initialized,
766 std::memory_order_relaxed)) {
767 KMP_FATAL(LockIsUninitialized, func);
769 if (lck->lk.self != lck) {
770 KMP_FATAL(LockIsUninitialized, func);
772 if (__kmp_is_ticket_lock_nestable(lck)) {
773 KMP_FATAL(LockNestableUsedAsSimple, func);
775 if (__kmp_get_ticket_lock_owner(lck) == -1) {
776 KMP_FATAL(LockUnsettingFree, func);
778 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
779 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
780 KMP_FATAL(LockUnsettingSetByAnother, func);
782 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
783 return __kmp_release_ticket_lock(lck, gtid);
786 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
787 lck->lk.location = NULL;
789 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
790 std::memory_order_relaxed);
791 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
792 std::memory_order_relaxed);
793 std::atomic_store_explicit(
794 &lck->lk.owner_id, 0,
795 std::memory_order_relaxed);
796 std::atomic_store_explicit(
797 &lck->lk.depth_locked, -1,
798 std::memory_order_relaxed);
799 std::atomic_store_explicit(&lck->lk.initialized,
true,
800 std::memory_order_release);
803 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
804 __kmp_init_ticket_lock(lck);
807 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
808 std::atomic_store_explicit(&lck->lk.initialized,
false,
809 std::memory_order_release);
811 lck->lk.location = NULL;
812 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
813 std::memory_order_relaxed);
814 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
815 std::memory_order_relaxed);
816 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
817 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
818 std::memory_order_relaxed);
821 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
822 char const *
const func =
"omp_destroy_lock";
824 if (!std::atomic_load_explicit(&lck->lk.initialized,
825 std::memory_order_relaxed)) {
826 KMP_FATAL(LockIsUninitialized, func);
828 if (lck->lk.self != lck) {
829 KMP_FATAL(LockIsUninitialized, func);
831 if (__kmp_is_ticket_lock_nestable(lck)) {
832 KMP_FATAL(LockNestableUsedAsSimple, func);
834 if (__kmp_get_ticket_lock_owner(lck) != -1) {
835 KMP_FATAL(LockStillOwned, func);
837 __kmp_destroy_ticket_lock(lck);
842 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
843 KMP_DEBUG_ASSERT(gtid >= 0);
845 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
846 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
847 std::memory_order_relaxed);
848 return KMP_LOCK_ACQUIRED_NEXT;
850 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
851 ANNOTATE_TICKET_ACQUIRED(lck);
852 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
853 std::memory_order_relaxed);
854 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
855 std::memory_order_relaxed);
856 return KMP_LOCK_ACQUIRED_FIRST;
860 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
862 char const *
const func =
"omp_set_nest_lock";
864 if (!std::atomic_load_explicit(&lck->lk.initialized,
865 std::memory_order_relaxed)) {
866 KMP_FATAL(LockIsUninitialized, func);
868 if (lck->lk.self != lck) {
869 KMP_FATAL(LockIsUninitialized, func);
871 if (!__kmp_is_ticket_lock_nestable(lck)) {
872 KMP_FATAL(LockSimpleUsedAsNestable, func);
874 return __kmp_acquire_nested_ticket_lock(lck, gtid);
877 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
880 KMP_DEBUG_ASSERT(gtid >= 0);
882 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
883 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
884 std::memory_order_relaxed) +
886 }
else if (!__kmp_test_ticket_lock(lck, gtid)) {
889 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
890 std::memory_order_relaxed);
891 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
892 std::memory_order_relaxed);
898 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
900 char const *
const func =
"omp_test_nest_lock";
902 if (!std::atomic_load_explicit(&lck->lk.initialized,
903 std::memory_order_relaxed)) {
904 KMP_FATAL(LockIsUninitialized, func);
906 if (lck->lk.self != lck) {
907 KMP_FATAL(LockIsUninitialized, func);
909 if (!__kmp_is_ticket_lock_nestable(lck)) {
910 KMP_FATAL(LockSimpleUsedAsNestable, func);
912 return __kmp_test_nested_ticket_lock(lck, gtid);
915 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
916 KMP_DEBUG_ASSERT(gtid >= 0);
918 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
919 std::memory_order_relaxed) -
921 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
922 __kmp_release_ticket_lock(lck, gtid);
923 return KMP_LOCK_RELEASED;
925 return KMP_LOCK_STILL_HELD;
928 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
930 char const *
const func =
"omp_unset_nest_lock";
932 if (!std::atomic_load_explicit(&lck->lk.initialized,
933 std::memory_order_relaxed)) {
934 KMP_FATAL(LockIsUninitialized, func);
936 if (lck->lk.self != lck) {
937 KMP_FATAL(LockIsUninitialized, func);
939 if (!__kmp_is_ticket_lock_nestable(lck)) {
940 KMP_FATAL(LockSimpleUsedAsNestable, func);
942 if (__kmp_get_ticket_lock_owner(lck) == -1) {
943 KMP_FATAL(LockUnsettingFree, func);
945 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
946 KMP_FATAL(LockUnsettingSetByAnother, func);
948 return __kmp_release_nested_ticket_lock(lck, gtid);
951 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
952 __kmp_init_ticket_lock(lck);
953 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
954 std::memory_order_relaxed);
958 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
959 __kmp_init_nested_ticket_lock(lck);
962 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
963 __kmp_destroy_ticket_lock(lck);
964 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
965 std::memory_order_relaxed);
969 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
970 char const *
const func =
"omp_destroy_nest_lock";
972 if (!std::atomic_load_explicit(&lck->lk.initialized,
973 std::memory_order_relaxed)) {
974 KMP_FATAL(LockIsUninitialized, func);
976 if (lck->lk.self != lck) {
977 KMP_FATAL(LockIsUninitialized, func);
979 if (!__kmp_is_ticket_lock_nestable(lck)) {
980 KMP_FATAL(LockSimpleUsedAsNestable, func);
982 if (__kmp_get_ticket_lock_owner(lck) != -1) {
983 KMP_FATAL(LockStillOwned, func);
985 __kmp_destroy_nested_ticket_lock(lck);
990 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
991 return std::atomic_load_explicit(&lck->lk.initialized,
992 std::memory_order_relaxed) &&
993 (lck->lk.self == lck);
996 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
997 return lck->lk.location;
1000 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
1002 lck->lk.location = loc;
1005 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
1006 return lck->lk.flags;
1009 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
1010 kmp_lock_flags_t flags) {
1011 lck->lk.flags = flags;
1069 #ifdef DEBUG_QUEUING_LOCKS 1072 #define TRACE_BUF_ELE 1024 1073 static char traces[TRACE_BUF_ELE][128] = {0};
1075 #define TRACE_LOCK(X, Y) \ 1076 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y); 1077 #define TRACE_LOCK_T(X, Y, Z) \ 1078 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X, Y, Z); 1079 #define TRACE_LOCK_HT(X, Y, Z, Q) \ 1080 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, \ 1083 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid,
1084 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1085 kmp_int32 tail_id) {
1088 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n");
1090 i = tc % TRACE_BUF_ELE;
1091 __kmp_printf_no_lock(
"%s\n", traces[i]);
1092 i = (i + 1) % TRACE_BUF_ELE;
1093 while (i != (tc % TRACE_BUF_ELE)) {
1094 __kmp_printf_no_lock(
"%s", traces[i]);
1095 i = (i + 1) % TRACE_BUF_ELE;
1097 __kmp_printf_no_lock(
"\n");
1099 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, " 1100 "next_wait:%d, head_id:%d, tail_id:%d\n",
1101 gtid + 1, this_thr->th.th_spin_here,
1102 this_thr->th.th_next_waiting, head_id, tail_id);
1104 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id);
1106 if (lck->lk.head_id >= 1) {
1107 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1109 __kmp_printf_no_lock(
"-> %d ", t);
1110 t = __kmp_threads[t - 1]->th.th_next_waiting;
1113 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id);
1114 __kmp_printf_no_lock(
"\n\n");
1119 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1120 return TCR_4(lck->lk.owner_id) - 1;
1123 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1124 return lck->lk.depth_locked != -1;
1128 template <
bool takeTime>
1131 __forceinline
static int 1132 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1134 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1135 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1136 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1137 volatile kmp_uint32 *spin_here_p;
1138 kmp_int32 need_mf = 1;
1141 omp_state_t prev_state = omp_state_undefined;
1145 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1147 KMP_FSYNC_PREPARE(lck);
1148 KMP_DEBUG_ASSERT(this_thr != NULL);
1149 spin_here_p = &this_thr->th.th_spin_here;
1151 #ifdef DEBUG_QUEUING_LOCKS 1152 TRACE_LOCK(gtid + 1,
"acq ent");
1154 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1155 if (this_thr->th.th_next_waiting != 0)
1156 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1158 KMP_DEBUG_ASSERT(!*spin_here_p);
1159 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1168 *spin_here_p = TRUE;
1180 #ifdef DEBUG_QUEUING_LOCKS 1182 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1191 enqueued = KMP_COMPARE_AND_STORE_ACQ64((
volatile kmp_int64 *)tail_id_p,
1193 KMP_PACK_64(gtid + 1, gtid + 1));
1194 #ifdef DEBUG_QUEUING_LOCKS 1196 TRACE_LOCK(gtid + 1,
"acq enq: (-1,0)->(tid,tid)");
1202 KMP_DEBUG_ASSERT(tail != gtid + 1);
1204 #ifdef DEBUG_QUEUING_LOCKS 1205 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1213 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1);
1215 #ifdef DEBUG_QUEUING_LOCKS 1217 TRACE_LOCK(gtid + 1,
"acq enq: (h,t)->(h,tid)");
1224 kmp_int32 grabbed_lock;
1226 #ifdef DEBUG_QUEUING_LOCKS 1228 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1234 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1);
1238 *spin_here_p = FALSE;
1242 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1244 #ifdef DEBUG_QUEUING_LOCKS 1245 TRACE_LOCK_HT(gtid + 1,
"acq exit: ", head, 0);
1249 if (ompt_enabled.enabled && prev_state != omp_state_undefined) {
1251 this_thr->th.ompt_thread_info.state = prev_state;
1252 this_thr->th.ompt_thread_info.wait_id = 0;
1256 KMP_FSYNC_ACQUIRED(lck);
1257 return KMP_LOCK_ACQUIRED_FIRST;
1264 if (ompt_enabled.enabled && prev_state == omp_state_undefined) {
1266 prev_state = this_thr->th.ompt_thread_info.state;
1267 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1268 this_thr->th.ompt_thread_info.state = omp_state_wait_lock;
1274 kmp_info_t *tail_thr = __kmp_thread_from_gtid(tail - 1);
1275 KMP_ASSERT(tail_thr != NULL);
1276 tail_thr->th.th_next_waiting = gtid + 1;
1280 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1286 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1288 #ifdef DEBUG_QUEUING_LOCKS 1289 TRACE_LOCK(gtid + 1,
"acq spin");
1291 if (this_thr->th.th_next_waiting != 0)
1292 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1294 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1295 KA_TRACE(1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after " 1296 "waiting on queue\n",
1299 #ifdef DEBUG_QUEUING_LOCKS 1300 TRACE_LOCK(gtid + 1,
"acq exit 2");
1305 this_thr->th.ompt_thread_info.state = prev_state;
1306 this_thr->th.ompt_thread_info.wait_id = 0;
1310 return KMP_LOCK_ACQUIRED_FIRST;
1316 KMP_YIELD(TCR_4(__kmp_nth) >
1317 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
1318 #ifdef DEBUG_QUEUING_LOCKS 1319 TRACE_LOCK(gtid + 1,
"acq retry");
1322 KMP_ASSERT2(0,
"should not get here");
1323 return KMP_LOCK_ACQUIRED_FIRST;
1326 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1327 KMP_DEBUG_ASSERT(gtid >= 0);
1329 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1330 ANNOTATE_QUEUING_ACQUIRED(lck);
1334 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1336 char const *
const func =
"omp_set_lock";
1337 if (lck->lk.initialized != lck) {
1338 KMP_FATAL(LockIsUninitialized, func);
1340 if (__kmp_is_queuing_lock_nestable(lck)) {
1341 KMP_FATAL(LockNestableUsedAsSimple, func);
1343 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1344 KMP_FATAL(LockIsAlreadyOwned, func);
1347 __kmp_acquire_queuing_lock(lck, gtid);
1349 lck->lk.owner_id = gtid + 1;
1350 return KMP_LOCK_ACQUIRED_FIRST;
1353 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1354 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1357 kmp_info_t *this_thr;
1360 KA_TRACE(1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid));
1361 KMP_DEBUG_ASSERT(gtid >= 0);
1363 this_thr = __kmp_thread_from_gtid(gtid);
1364 KMP_DEBUG_ASSERT(this_thr != NULL);
1365 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1372 if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
1374 (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
1375 KMP_FSYNC_ACQUIRED(lck);
1376 ANNOTATE_QUEUING_ACQUIRED(lck);
1382 (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
1386 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1388 char const *
const func =
"omp_test_lock";
1389 if (lck->lk.initialized != lck) {
1390 KMP_FATAL(LockIsUninitialized, func);
1392 if (__kmp_is_queuing_lock_nestable(lck)) {
1393 KMP_FATAL(LockNestableUsedAsSimple, func);
1396 int retval = __kmp_test_queuing_lock(lck, gtid);
1399 lck->lk.owner_id = gtid + 1;
1404 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1405 kmp_info_t *this_thr;
1406 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1407 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1410 (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1411 KMP_DEBUG_ASSERT(gtid >= 0);
1412 this_thr = __kmp_thread_from_gtid(gtid);
1413 KMP_DEBUG_ASSERT(this_thr != NULL);
1414 #ifdef DEBUG_QUEUING_LOCKS 1415 TRACE_LOCK(gtid + 1,
"rel ent");
1417 if (this_thr->th.th_spin_here)
1418 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1419 if (this_thr->th.th_next_waiting != 0)
1420 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1422 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1423 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1425 KMP_FSYNC_RELEASING(lck);
1426 ANNOTATE_QUEUING_RELEASED(lck);
1435 #ifdef DEBUG_QUEUING_LOCKS 1437 TRACE_LOCK_HT(gtid + 1,
"rel read: ", head, tail);
1439 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1441 KMP_DEBUG_ASSERT(head !=
1446 if (KMP_COMPARE_AND_STORE_REL32(head_id_p, -1, 0)) {
1449 (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1451 #ifdef DEBUG_QUEUING_LOCKS 1452 TRACE_LOCK_HT(gtid + 1,
"rel exit: ", 0, 0);
1458 return KMP_LOCK_RELEASED;
1465 #ifdef DEBUG_QUEUING_LOCKS 1467 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1469 KMP_DEBUG_ASSERT(head > 0);
1472 dequeued = KMP_COMPARE_AND_STORE_REL64(
1473 RCAST(
volatile kmp_int64 *, tail_id_p), KMP_PACK_64(head, head),
1474 KMP_PACK_64(-1, 0));
1475 #ifdef DEBUG_QUEUING_LOCKS 1476 TRACE_LOCK(gtid + 1,
"rel deq: (h,h)->(-1,0)");
1480 volatile kmp_int32 *waiting_id_p;
1481 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1482 KMP_DEBUG_ASSERT(head_thr != NULL);
1483 waiting_id_p = &head_thr->th.th_next_waiting;
1486 #ifdef DEBUG_QUEUING_LOCKS 1487 if (head <= 0 || tail <= 0)
1488 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1490 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1496 *head_id_p = KMP_WAIT_YIELD((
volatile kmp_uint32 *)waiting_id_p, 0,
1498 #ifdef DEBUG_QUEUING_LOCKS 1499 TRACE_LOCK(gtid + 1,
"rel deq: (h,t)->(h',t)");
1506 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1507 KMP_DEBUG_ASSERT(head_thr != NULL);
1510 #ifdef DEBUG_QUEUING_LOCKS 1511 if (head <= 0 || tail <= 0)
1512 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1514 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1518 head_thr->th.th_next_waiting = 0;
1519 #ifdef DEBUG_QUEUING_LOCKS 1520 TRACE_LOCK_T(gtid + 1,
"rel nw=0 for t=", head);
1525 head_thr->th.th_spin_here = FALSE;
1527 KA_TRACE(1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after " 1530 #ifdef DEBUG_QUEUING_LOCKS 1531 TRACE_LOCK(gtid + 1,
"rel exit 2");
1533 return KMP_LOCK_RELEASED;
1538 #ifdef DEBUG_QUEUING_LOCKS 1539 TRACE_LOCK(gtid + 1,
"rel retry");
1543 KMP_ASSERT2(0,
"should not get here");
1544 return KMP_LOCK_RELEASED;
1547 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1549 char const *
const func =
"omp_unset_lock";
1551 if (lck->lk.initialized != lck) {
1552 KMP_FATAL(LockIsUninitialized, func);
1554 if (__kmp_is_queuing_lock_nestable(lck)) {
1555 KMP_FATAL(LockNestableUsedAsSimple, func);
1557 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1558 KMP_FATAL(LockUnsettingFree, func);
1560 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1561 KMP_FATAL(LockUnsettingSetByAnother, func);
1563 lck->lk.owner_id = 0;
1564 return __kmp_release_queuing_lock(lck, gtid);
1567 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1568 lck->lk.location = NULL;
1569 lck->lk.head_id = 0;
1570 lck->lk.tail_id = 0;
1571 lck->lk.next_ticket = 0;
1572 lck->lk.now_serving = 0;
1573 lck->lk.owner_id = 0;
1574 lck->lk.depth_locked = -1;
1575 lck->lk.initialized = lck;
1577 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1580 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1581 __kmp_init_queuing_lock(lck);
1584 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1585 lck->lk.initialized = NULL;
1586 lck->lk.location = NULL;
1587 lck->lk.head_id = 0;
1588 lck->lk.tail_id = 0;
1589 lck->lk.next_ticket = 0;
1590 lck->lk.now_serving = 0;
1591 lck->lk.owner_id = 0;
1592 lck->lk.depth_locked = -1;
1595 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1596 char const *
const func =
"omp_destroy_lock";
1597 if (lck->lk.initialized != lck) {
1598 KMP_FATAL(LockIsUninitialized, func);
1600 if (__kmp_is_queuing_lock_nestable(lck)) {
1601 KMP_FATAL(LockNestableUsedAsSimple, func);
1603 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1604 KMP_FATAL(LockStillOwned, func);
1606 __kmp_destroy_queuing_lock(lck);
1611 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1612 KMP_DEBUG_ASSERT(gtid >= 0);
1614 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1615 lck->lk.depth_locked += 1;
1616 return KMP_LOCK_ACQUIRED_NEXT;
1618 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1619 ANNOTATE_QUEUING_ACQUIRED(lck);
1621 lck->lk.depth_locked = 1;
1623 lck->lk.owner_id = gtid + 1;
1624 return KMP_LOCK_ACQUIRED_FIRST;
1629 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1631 char const *
const func =
"omp_set_nest_lock";
1632 if (lck->lk.initialized != lck) {
1633 KMP_FATAL(LockIsUninitialized, func);
1635 if (!__kmp_is_queuing_lock_nestable(lck)) {
1636 KMP_FATAL(LockSimpleUsedAsNestable, func);
1638 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1641 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1644 KMP_DEBUG_ASSERT(gtid >= 0);
1646 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1647 retval = ++lck->lk.depth_locked;
1648 }
else if (!__kmp_test_queuing_lock(lck, gtid)) {
1652 retval = lck->lk.depth_locked = 1;
1654 lck->lk.owner_id = gtid + 1;
1659 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1661 char const *
const func =
"omp_test_nest_lock";
1662 if (lck->lk.initialized != lck) {
1663 KMP_FATAL(LockIsUninitialized, func);
1665 if (!__kmp_is_queuing_lock_nestable(lck)) {
1666 KMP_FATAL(LockSimpleUsedAsNestable, func);
1668 return __kmp_test_nested_queuing_lock(lck, gtid);
1671 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1672 KMP_DEBUG_ASSERT(gtid >= 0);
1675 if (--(lck->lk.depth_locked) == 0) {
1677 lck->lk.owner_id = 0;
1678 __kmp_release_queuing_lock(lck, gtid);
1679 return KMP_LOCK_RELEASED;
1681 return KMP_LOCK_STILL_HELD;
1685 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1687 char const *
const func =
"omp_unset_nest_lock";
1689 if (lck->lk.initialized != lck) {
1690 KMP_FATAL(LockIsUninitialized, func);
1692 if (!__kmp_is_queuing_lock_nestable(lck)) {
1693 KMP_FATAL(LockSimpleUsedAsNestable, func);
1695 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1696 KMP_FATAL(LockUnsettingFree, func);
1698 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1699 KMP_FATAL(LockUnsettingSetByAnother, func);
1701 return __kmp_release_nested_queuing_lock(lck, gtid);
1704 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1705 __kmp_init_queuing_lock(lck);
1706 lck->lk.depth_locked = 0;
1710 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1711 __kmp_init_nested_queuing_lock(lck);
1714 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1715 __kmp_destroy_queuing_lock(lck);
1716 lck->lk.depth_locked = 0;
1720 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1721 char const *
const func =
"omp_destroy_nest_lock";
1722 if (lck->lk.initialized != lck) {
1723 KMP_FATAL(LockIsUninitialized, func);
1725 if (!__kmp_is_queuing_lock_nestable(lck)) {
1726 KMP_FATAL(LockSimpleUsedAsNestable, func);
1728 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1729 KMP_FATAL(LockStillOwned, func);
1731 __kmp_destroy_nested_queuing_lock(lck);
1736 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
1737 return lck == lck->lk.initialized;
1740 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1741 return lck->lk.location;
1744 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1746 lck->lk.location = loc;
1749 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1750 return lck->lk.flags;
1753 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1754 kmp_lock_flags_t flags) {
1755 lck->lk.flags = flags;
1758 #if KMP_USE_ADAPTIVE_LOCKS 1762 #if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1764 #include <immintrin.h> 1765 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1770 #define _XBEGIN_STARTED (~0u) 1771 #define _XABORT_EXPLICIT (1 << 0) 1772 #define _XABORT_RETRY (1 << 1) 1773 #define _XABORT_CONFLICT (1 << 2) 1774 #define _XABORT_CAPACITY (1 << 3) 1775 #define _XABORT_DEBUG (1 << 4) 1776 #define _XABORT_NESTED (1 << 5) 1777 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF)) 1780 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1782 #define STRINGIZE_INTERNAL(arg) #arg 1783 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg) 1789 static __inline
int _xbegin() {
1817 #endif // KMP_ARCH_X86_64 1826 __asm__
volatile(
"1: .byte 0xC7; .byte 0xF8;\n" 1829 "1: movl %%eax,%0\n" 1831 :
"+r"(res)::
"memory",
"%eax");
1832 #endif // KMP_OS_WINDOWS 1837 static __inline
void _xend() {
1845 __asm__
volatile(
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1854 #define _xabort(ARG) _asm _emit 0xc6 _asm _emit 0xf8 _asm _emit ARG 1856 #define _xabort(ARG) \ 1857 __asm__ volatile(".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG):::"memory"); 1860 #endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1863 #if KMP_DEBUG_ADAPTIVE_LOCKS 1868 static kmp_adaptive_lock_statistics_t destroyedStats;
1871 static kmp_adaptive_lock_info_t liveLocks;
1874 static kmp_bootstrap_lock_t chain_lock =
1875 KMP_BOOTSTRAP_LOCK_INITIALIZER(chain_lock);
1878 void __kmp_init_speculative_stats() {
1879 kmp_adaptive_lock_info_t *lck = &liveLocks;
1881 memset(CCAST(kmp_adaptive_lock_statistics_t *, &(lck->stats)), 0,
1882 sizeof(lck->stats));
1883 lck->stats.next = lck;
1884 lck->stats.prev = lck;
1886 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1887 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1889 __kmp_init_bootstrap_lock(&chain_lock);
1893 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1894 __kmp_acquire_bootstrap_lock(&chain_lock);
1896 lck->stats.next = liveLocks.stats.next;
1897 lck->stats.prev = &liveLocks;
1899 liveLocks.stats.next = lck;
1900 lck->stats.next->stats.prev = lck;
1902 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1903 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1905 __kmp_release_bootstrap_lock(&chain_lock);
1908 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1909 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1910 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1912 kmp_adaptive_lock_info_t *n = lck->stats.next;
1913 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1919 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1920 memset(CCAST(kmp_adaptive_lock_statistics_t *, &lck->stats), 0,
1921 sizeof(lck->stats));
1922 __kmp_remember_lock(lck);
1925 static void __kmp_add_stats(kmp_adaptive_lock_statistics_t *t,
1926 kmp_adaptive_lock_info_t *lck) {
1927 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
1929 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1930 t->successfulSpeculations += s->successfulSpeculations;
1931 t->hardFailedSpeculations += s->hardFailedSpeculations;
1932 t->softFailedSpeculations += s->softFailedSpeculations;
1933 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
1934 t->lemmingYields += s->lemmingYields;
1937 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1938 __kmp_acquire_bootstrap_lock(&chain_lock);
1940 __kmp_add_stats(&destroyedStats, lck);
1941 __kmp_forget_lock(lck);
1943 __kmp_release_bootstrap_lock(&chain_lock);
1946 static float percent(kmp_uint32 count, kmp_uint32 total) {
1947 return (total == 0) ? 0.0 : (100.0 * count) / total;
1950 static FILE *__kmp_open_stats_file() {
1951 if (strcmp(__kmp_speculative_statsfile,
"-") == 0)
1954 size_t buffLen = KMP_STRLEN(__kmp_speculative_statsfile) + 20;
1955 char buffer[buffLen];
1956 KMP_SNPRINTF(&buffer[0], buffLen, __kmp_speculative_statsfile,
1957 (kmp_int32)getpid());
1958 FILE *result = fopen(&buffer[0],
"w");
1961 return result ? result : stdout;
1964 void __kmp_print_speculative_stats() {
1965 kmp_adaptive_lock_statistics_t total = destroyedStats;
1966 kmp_adaptive_lock_info_t *lck;
1968 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1969 __kmp_add_stats(&total, lck);
1971 kmp_adaptive_lock_statistics_t *t = &total;
1972 kmp_uint32 totalSections =
1973 t->nonSpeculativeAcquires + t->successfulSpeculations;
1974 kmp_uint32 totalSpeculations = t->successfulSpeculations +
1975 t->hardFailedSpeculations +
1976 t->softFailedSpeculations;
1977 if (totalSections <= 0)
1980 FILE *statsFile = __kmp_open_stats_file();
1982 fprintf(statsFile,
"Speculative lock statistics (all approximate!)\n");
1983 fprintf(statsFile,
" Lock parameters: \n" 1984 " max_soft_retries : %10d\n" 1985 " max_badness : %10d\n",
1986 __kmp_adaptive_backoff_params.max_soft_retries,
1987 __kmp_adaptive_backoff_params.max_badness);
1988 fprintf(statsFile,
" Non-speculative acquire attempts : %10d\n",
1989 t->nonSpeculativeAcquireAttempts);
1990 fprintf(statsFile,
" Total critical sections : %10d\n",
1992 fprintf(statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
1993 t->successfulSpeculations,
1994 percent(t->successfulSpeculations, totalSections));
1995 fprintf(statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
1996 t->nonSpeculativeAcquires,
1997 percent(t->nonSpeculativeAcquires, totalSections));
1998 fprintf(statsFile,
" Lemming yields : %10d\n\n",
2001 fprintf(statsFile,
" Speculative acquire attempts : %10d\n",
2003 fprintf(statsFile,
" Successes : %10d (%5.1f%%)\n",
2004 t->successfulSpeculations,
2005 percent(t->successfulSpeculations, totalSpeculations));
2006 fprintf(statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2007 t->softFailedSpeculations,
2008 percent(t->softFailedSpeculations, totalSpeculations));
2009 fprintf(statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2010 t->hardFailedSpeculations,
2011 percent(t->hardFailedSpeculations, totalSpeculations));
2013 if (statsFile != stdout)
2017 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++) 2019 #define KMP_INC_STAT(lck, stat) 2021 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 2023 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
2026 bool res = lck->lk.head_id == 0;
2030 #if KMP_COMPILER_ICC 2033 __sync_synchronize();
2040 static __inline
void 2041 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
2043 lck->lk.adaptive.badness = 0;
2044 KMP_INC_STAT(lck, successfulSpeculations);
2048 static __inline
void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
2049 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
2050 if (newBadness > lck->lk.adaptive.max_badness) {
2053 lck->lk.adaptive.badness = newBadness;
2058 static __inline
int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
2060 kmp_uint32 badness = lck->lk.adaptive.badness;
2061 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
2062 int res = (attempts & badness) == 0;
2068 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
2070 int retries = lck->lk.adaptive.max_soft_retries;
2077 kmp_uint32 status = _xbegin();
2082 if (status == _XBEGIN_STARTED) {
2087 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2091 KMP_ASSERT2(0,
"should not get here");
2096 if (status & SOFT_ABORT_MASK) {
2097 KMP_INC_STAT(lck, softFailedSpeculations);
2100 KMP_INC_STAT(lck, hardFailedSpeculations);
2105 }
while (retries--);
2109 __kmp_step_badness(lck);
2116 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2118 if (__kmp_should_speculate(lck, gtid) &&
2119 __kmp_test_adaptive_lock_only(lck, gtid))
2124 lck->lk.adaptive.acquire_attempts++;
2127 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2128 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2135 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2137 char const *
const func =
"omp_test_lock";
2138 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2139 KMP_FATAL(LockIsUninitialized, func);
2142 int retval = __kmp_test_adaptive_lock(lck, gtid);
2145 lck->lk.qlk.owner_id = gtid + 1;
2161 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2163 if (__kmp_should_speculate(lck, gtid)) {
2164 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2165 if (__kmp_test_adaptive_lock_only(lck, gtid))
2174 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2175 KMP_INC_STAT(lck, lemmingYields);
2179 if (__kmp_test_adaptive_lock_only(lck, gtid))
2186 lck->lk.adaptive.acquire_attempts++;
2188 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2190 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2191 ANNOTATE_QUEUING_ACQUIRED(lck);
2194 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2196 char const *
const func =
"omp_set_lock";
2197 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2198 KMP_FATAL(LockIsUninitialized, func);
2200 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2201 KMP_FATAL(LockIsAlreadyOwned, func);
2204 __kmp_acquire_adaptive_lock(lck, gtid);
2206 lck->lk.qlk.owner_id = gtid + 1;
2209 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2211 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
2216 __kmp_update_badness_after_success(lck);
2219 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2221 return KMP_LOCK_RELEASED;
2224 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2226 char const *
const func =
"omp_unset_lock";
2228 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2229 KMP_FATAL(LockIsUninitialized, func);
2231 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2232 KMP_FATAL(LockUnsettingFree, func);
2234 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2235 KMP_FATAL(LockUnsettingSetByAnother, func);
2237 lck->lk.qlk.owner_id = 0;
2238 __kmp_release_adaptive_lock(lck, gtid);
2239 return KMP_LOCK_RELEASED;
2242 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2243 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2244 lck->lk.adaptive.badness = 0;
2245 lck->lk.adaptive.acquire_attempts = 0;
2246 lck->lk.adaptive.max_soft_retries =
2247 __kmp_adaptive_backoff_params.max_soft_retries;
2248 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2249 #if KMP_DEBUG_ADAPTIVE_LOCKS 2250 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2252 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2255 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2256 __kmp_init_adaptive_lock(lck);
2259 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2260 #if KMP_DEBUG_ADAPTIVE_LOCKS 2261 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2263 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2267 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2268 char const *
const func =
"omp_destroy_lock";
2269 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2270 KMP_FATAL(LockIsUninitialized, func);
2272 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2273 KMP_FATAL(LockStillOwned, func);
2275 __kmp_destroy_adaptive_lock(lck);
2278 #endif // KMP_USE_ADAPTIVE_LOCKS 2284 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2285 return lck->lk.owner_id - 1;
2288 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2289 return lck->lk.depth_locked != -1;
2292 __forceinline
static int 2293 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2294 kmp_uint64 ticket = KMP_ATOMIC_INC(&lck->lk.next_ticket);
2295 kmp_uint64 mask = lck->lk.mask;
2296 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2298 #ifdef USE_LOCK_PROFILE 2299 if (polls[ticket & mask] != ticket)
2300 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2314 KMP_FSYNC_PREPARE(lck);
2315 KMP_INIT_YIELD(spins);
2316 while (polls[ticket & mask] < ticket) {
2321 KMP_YIELD(TCR_4(__kmp_nth) >
2322 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2323 KMP_YIELD_SPIN(spins);
2332 mask = lck->lk.mask;
2333 polls = lck->lk.polls;
2337 KMP_FSYNC_ACQUIRED(lck);
2338 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2340 lck->lk.now_serving = ticket;
2347 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2348 __kmp_free(lck->lk.old_polls);
2349 lck->lk.old_polls = NULL;
2350 lck->lk.cleanup_ticket = 0;
2356 if (lck->lk.old_polls == NULL) {
2357 bool reconfigure =
false;
2358 std::atomic<kmp_uint64> *old_polls = polls;
2359 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2361 if (TCR_4(__kmp_nth) >
2362 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2365 if (num_polls > 1) {
2367 num_polls = TCR_4(lck->lk.num_polls);
2370 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2378 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2379 if (num_waiting > num_polls) {
2380 kmp_uint32 old_num_polls = num_polls;
2383 mask = (mask << 1) | 1;
2385 }
while (num_polls <= num_waiting);
2391 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2394 for (i = 0; i < old_num_polls; i++) {
2395 polls[i].store(old_polls[i]);
2410 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring " 2411 "lock %p to %d polls\n",
2412 ticket, lck, num_polls));
2414 lck->lk.old_polls = old_polls;
2415 lck->lk.polls = polls;
2419 lck->lk.num_polls = num_polls;
2420 lck->lk.mask = mask;
2428 lck->lk.cleanup_ticket = lck->lk.next_ticket;
2431 return KMP_LOCK_ACQUIRED_FIRST;
2434 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2435 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2436 ANNOTATE_DRDPA_ACQUIRED(lck);
2440 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2442 char const *
const func =
"omp_set_lock";
2443 if (lck->lk.initialized != lck) {
2444 KMP_FATAL(LockIsUninitialized, func);
2446 if (__kmp_is_drdpa_lock_nestable(lck)) {
2447 KMP_FATAL(LockNestableUsedAsSimple, func);
2449 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2450 KMP_FATAL(LockIsAlreadyOwned, func);
2453 __kmp_acquire_drdpa_lock(lck, gtid);
2455 lck->lk.owner_id = gtid + 1;
2456 return KMP_LOCK_ACQUIRED_FIRST;
2459 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2462 kmp_uint64 ticket = lck->lk.next_ticket;
2463 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2464 kmp_uint64 mask = lck->lk.mask;
2465 if (polls[ticket & mask] == ticket) {
2466 kmp_uint64 next_ticket = ticket + 1;
2467 if (__kmp_atomic_compare_store_acq(&lck->lk.next_ticket, ticket,
2469 KMP_FSYNC_ACQUIRED(lck);
2470 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2472 lck->lk.now_serving = ticket;
2486 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2488 char const *
const func =
"omp_test_lock";
2489 if (lck->lk.initialized != lck) {
2490 KMP_FATAL(LockIsUninitialized, func);
2492 if (__kmp_is_drdpa_lock_nestable(lck)) {
2493 KMP_FATAL(LockNestableUsedAsSimple, func);
2496 int retval = __kmp_test_drdpa_lock(lck, gtid);
2499 lck->lk.owner_id = gtid + 1;
2504 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2507 kmp_uint64 ticket = lck->lk.now_serving + 1;
2508 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2509 kmp_uint64 mask = lck->lk.mask;
2510 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2512 KMP_FSYNC_RELEASING(lck);
2513 ANNOTATE_DRDPA_RELEASED(lck);
2514 polls[ticket & mask] = ticket;
2515 return KMP_LOCK_RELEASED;
2518 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2520 char const *
const func =
"omp_unset_lock";
2522 if (lck->lk.initialized != lck) {
2523 KMP_FATAL(LockIsUninitialized, func);
2525 if (__kmp_is_drdpa_lock_nestable(lck)) {
2526 KMP_FATAL(LockNestableUsedAsSimple, func);
2528 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2529 KMP_FATAL(LockUnsettingFree, func);
2531 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2532 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2533 KMP_FATAL(LockUnsettingSetByAnother, func);
2535 lck->lk.owner_id = 0;
2536 return __kmp_release_drdpa_lock(lck, gtid);
2539 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2540 lck->lk.location = NULL;
2542 lck->lk.num_polls = 1;
2543 lck->lk.polls = (std::atomic<kmp_uint64> *)__kmp_allocate(
2544 lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2545 lck->lk.cleanup_ticket = 0;
2546 lck->lk.old_polls = NULL;
2547 lck->lk.next_ticket = 0;
2548 lck->lk.now_serving = 0;
2549 lck->lk.owner_id = 0;
2550 lck->lk.depth_locked = -1;
2551 lck->lk.initialized = lck;
2553 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2556 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2557 __kmp_init_drdpa_lock(lck);
2560 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2561 lck->lk.initialized = NULL;
2562 lck->lk.location = NULL;
2563 if (lck->lk.polls.load() != NULL) {
2564 __kmp_free(lck->lk.polls.load());
2565 lck->lk.polls = NULL;
2567 if (lck->lk.old_polls != NULL) {
2568 __kmp_free(lck->lk.old_polls);
2569 lck->lk.old_polls = NULL;
2572 lck->lk.num_polls = 0;
2573 lck->lk.cleanup_ticket = 0;
2574 lck->lk.next_ticket = 0;
2575 lck->lk.now_serving = 0;
2576 lck->lk.owner_id = 0;
2577 lck->lk.depth_locked = -1;
2580 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2581 char const *
const func =
"omp_destroy_lock";
2582 if (lck->lk.initialized != lck) {
2583 KMP_FATAL(LockIsUninitialized, func);
2585 if (__kmp_is_drdpa_lock_nestable(lck)) {
2586 KMP_FATAL(LockNestableUsedAsSimple, func);
2588 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2589 KMP_FATAL(LockStillOwned, func);
2591 __kmp_destroy_drdpa_lock(lck);
2596 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2597 KMP_DEBUG_ASSERT(gtid >= 0);
2599 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2600 lck->lk.depth_locked += 1;
2601 return KMP_LOCK_ACQUIRED_NEXT;
2603 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2604 ANNOTATE_DRDPA_ACQUIRED(lck);
2606 lck->lk.depth_locked = 1;
2608 lck->lk.owner_id = gtid + 1;
2609 return KMP_LOCK_ACQUIRED_FIRST;
2613 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2615 char const *
const func =
"omp_set_nest_lock";
2616 if (lck->lk.initialized != lck) {
2617 KMP_FATAL(LockIsUninitialized, func);
2619 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2620 KMP_FATAL(LockSimpleUsedAsNestable, func);
2622 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2625 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2628 KMP_DEBUG_ASSERT(gtid >= 0);
2630 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2631 retval = ++lck->lk.depth_locked;
2632 }
else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2636 retval = lck->lk.depth_locked = 1;
2638 lck->lk.owner_id = gtid + 1;
2643 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2645 char const *
const func =
"omp_test_nest_lock";
2646 if (lck->lk.initialized != lck) {
2647 KMP_FATAL(LockIsUninitialized, func);
2649 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2650 KMP_FATAL(LockSimpleUsedAsNestable, func);
2652 return __kmp_test_nested_drdpa_lock(lck, gtid);
2655 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2656 KMP_DEBUG_ASSERT(gtid >= 0);
2659 if (--(lck->lk.depth_locked) == 0) {
2661 lck->lk.owner_id = 0;
2662 __kmp_release_drdpa_lock(lck, gtid);
2663 return KMP_LOCK_RELEASED;
2665 return KMP_LOCK_STILL_HELD;
2668 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2670 char const *
const func =
"omp_unset_nest_lock";
2672 if (lck->lk.initialized != lck) {
2673 KMP_FATAL(LockIsUninitialized, func);
2675 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2676 KMP_FATAL(LockSimpleUsedAsNestable, func);
2678 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2679 KMP_FATAL(LockUnsettingFree, func);
2681 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2682 KMP_FATAL(LockUnsettingSetByAnother, func);
2684 return __kmp_release_nested_drdpa_lock(lck, gtid);
2687 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2688 __kmp_init_drdpa_lock(lck);
2689 lck->lk.depth_locked = 0;
2692 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2693 __kmp_init_nested_drdpa_lock(lck);
2696 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2697 __kmp_destroy_drdpa_lock(lck);
2698 lck->lk.depth_locked = 0;
2701 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2702 char const *
const func =
"omp_destroy_nest_lock";
2703 if (lck->lk.initialized != lck) {
2704 KMP_FATAL(LockIsUninitialized, func);
2706 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2707 KMP_FATAL(LockSimpleUsedAsNestable, func);
2709 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2710 KMP_FATAL(LockStillOwned, func);
2712 __kmp_destroy_nested_drdpa_lock(lck);
2717 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
2718 return lck == lck->lk.initialized;
2721 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2722 return lck->lk.location;
2725 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2727 lck->lk.location = loc;
2730 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2731 return lck->lk.flags;
2734 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2735 kmp_lock_flags_t flags) {
2736 lck->lk.flags = flags;
2740 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2741 #define __kmp_tsc() __kmp_hardware_timestamp() 2743 kmp_backoff_t __kmp_spin_backoff_params = {1, 4096, 100};
2746 extern kmp_uint64 __kmp_now_nsec();
2747 kmp_backoff_t __kmp_spin_backoff_params = {1, 256, 100};
2748 #define __kmp_tsc() __kmp_now_nsec() 2758 static inline bool before(kmp_uint64 a, kmp_uint64 b) {
2759 return ((kmp_int64)b - (kmp_int64)a) > 0;
2763 void __kmp_spin_backoff(kmp_backoff_t *boff) {
2766 for (i = boff->step; i > 0; i--) {
2767 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
2770 }
while (before(__kmp_tsc(), goal));
2772 boff->step = (boff->step << 1 | 1) & (boff->max_backoff - 1);
2775 #if KMP_USE_DYNAMIC_LOCK 2779 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2780 kmp_dyna_lockseq_t seq) {
2781 TCW_4(*lck, KMP_GET_D_TAG(seq));
2784 (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
2790 #define HLE_ACQUIRE ".byte 0xf2;" 2791 #define HLE_RELEASE ".byte 0xf3;" 2793 static inline kmp_uint32 swap4(kmp_uint32
volatile *p, kmp_uint32 v) {
2794 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" :
"+r"(v),
"+m"(*p) : :
"memory");
2798 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2800 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2802 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2805 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
2806 for (
int i = delay; i != 0; --i)
2808 delay = ((delay << 1) | 1) & 7;
2810 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2814 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2816 __kmp_acquire_hle_lock(lck, gtid);
2819 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2820 __asm__
volatile(HLE_RELEASE
"movl %1,%0" 2822 :
"r"(KMP_LOCK_FREE(hle))
2824 return KMP_LOCK_RELEASED;
2827 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2829 return __kmp_release_hle_lock(lck, gtid);
2832 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2833 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2836 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2838 return __kmp_test_hle_lock(lck, gtid);
2841 static void __kmp_init_rtm_lock(kmp_queuing_lock_t *lck) {
2842 __kmp_init_queuing_lock(lck);
2845 static void __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck) {
2846 __kmp_destroy_queuing_lock(lck);
2849 static void __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2850 unsigned retries = 3, status;
2853 if (status == _XBEGIN_STARTED) {
2854 if (__kmp_is_unlocked_queuing_lock(lck))
2858 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2860 while (!__kmp_is_unlocked_queuing_lock(lck))
2862 }
else if (!(status & _XABORT_RETRY))
2864 }
while (retries--);
2867 __kmp_acquire_queuing_lock(lck, gtid);
2870 static void __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2872 __kmp_acquire_rtm_lock(lck, gtid);
2875 static int __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2876 if (__kmp_is_unlocked_queuing_lock(lck)) {
2881 __kmp_release_queuing_lock(lck, gtid);
2883 return KMP_LOCK_RELEASED;
2886 static int __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2888 return __kmp_release_rtm_lock(lck, gtid);
2891 static int __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2892 unsigned retries = 3, status;
2895 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2898 if (!(status & _XABORT_RETRY))
2900 }
while (retries--);
2902 return (__kmp_is_unlocked_queuing_lock(lck)) ? 1 : 0;
2905 static int __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2907 return __kmp_test_rtm_lock(lck, gtid);
2910 #endif // KMP_USE_TSX 2913 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *l,
2914 kmp_dyna_lockseq_t tag);
2915 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock);
2916 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2917 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2918 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2919 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2921 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2923 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2930 #define expand(l, op) 0, __kmp_init_direct_lock, 2931 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t) = {
2932 __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init)};
2936 #define expand(l, op) 0, (void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock, 2937 void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *) = {
2938 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2942 #define expand(l, op) \ 2943 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 2944 static int (*direct_set[])(kmp_dyna_lock_t *, kmp_int32) = {
2945 __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire)};
2947 #define expand(l, op) \ 2948 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 2949 static int (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2950 __kmp_set_indirect_lock_with_checks, 0,
2951 KMP_FOREACH_D_LOCK(expand, acquire)};
2955 #define expand(l, op) \ 2956 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 2957 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32) = {
2958 __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release)};
2959 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32) = {
2960 __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test)};
2962 #define expand(l, op) \ 2963 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 2964 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2965 __kmp_unset_indirect_lock_with_checks, 0,
2966 KMP_FOREACH_D_LOCK(expand, release)};
2967 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2968 __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test)};
2972 int (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
2973 int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
2974 int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
2977 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock, 2978 void (*__kmp_indirect_init[])(kmp_user_lock_p) = {
2979 KMP_FOREACH_I_LOCK(expand, init)};
2980 void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = {
2981 KMP_FOREACH_I_LOCK(expand, destroy)};
2985 #define expand(l, op) \ 2986 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock, 2987 static int (*indirect_set[])(kmp_user_lock_p,
2988 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, acquire)};
2990 #define expand(l, op) \ 2991 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks, 2992 static int (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = {
2993 KMP_FOREACH_I_LOCK(expand, acquire)};
2997 #define expand(l, op) \ 2998 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock, 2999 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = {
3000 KMP_FOREACH_I_LOCK(expand, release)};
3001 static int (*indirect_test[])(kmp_user_lock_p,
3002 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, test)};
3004 #define expand(l, op) \ 3005 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks, 3006 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = {
3007 KMP_FOREACH_I_LOCK(expand, release)};
3008 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = {
3009 KMP_FOREACH_I_LOCK(expand, test)};
3013 int (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
3014 int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
3015 int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
3018 kmp_indirect_lock_table_t __kmp_i_lock_table;
3021 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {0};
3024 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3026 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3027 kmp_lock_flags_t) = {0};
3028 const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
3029 kmp_user_lock_p) = {0};
3030 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
3031 kmp_user_lock_p) = {0};
3034 static kmp_indirect_lock_t *__kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = {0};
3041 kmp_indirect_lock_t *__kmp_allocate_indirect_lock(
void **user_lock,
3043 kmp_indirect_locktag_t tag) {
3044 kmp_indirect_lock_t *lck;
3045 kmp_lock_index_t idx;
3047 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3049 if (__kmp_indirect_lock_pool[tag] != NULL) {
3051 lck = __kmp_indirect_lock_pool[tag];
3052 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3053 idx = lck->lock->pool.index;
3054 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3055 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n",
3058 idx = __kmp_i_lock_table.next;
3060 if (idx == __kmp_i_lock_table.size) {
3062 int row = __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK;
3063 kmp_indirect_lock_t **new_table = (kmp_indirect_lock_t **)__kmp_allocate(
3064 2 * row *
sizeof(kmp_indirect_lock_t *));
3065 KMP_MEMCPY(new_table, __kmp_i_lock_table.table,
3066 row *
sizeof(kmp_indirect_lock_t *));
3067 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3068 __kmp_i_lock_table.table = new_table;
3069 __kmp_free(old_table);
3071 for (
int i = row; i < 2 * row; ++i)
3072 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)__kmp_allocate(
3073 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3074 __kmp_i_lock_table.size = 2 * idx;
3076 __kmp_i_lock_table.next++;
3077 lck = KMP_GET_I_LOCK(idx);
3079 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3081 (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3084 __kmp_release_lock(&__kmp_global_lock, gtid);
3088 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3089 *((kmp_lock_index_t *)user_lock) = idx
3092 *((kmp_indirect_lock_t **)user_lock) = lck;
3099 static __forceinline kmp_indirect_lock_t *
3100 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func) {
3101 if (__kmp_env_consistency_check) {
3102 kmp_indirect_lock_t *lck = NULL;
3103 if (user_lock == NULL) {
3104 KMP_FATAL(LockIsUninitialized, func);
3106 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3107 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3108 if (idx >= __kmp_i_lock_table.size) {
3109 KMP_FATAL(LockIsUninitialized, func);
3111 lck = KMP_GET_I_LOCK(idx);
3113 lck = *((kmp_indirect_lock_t **)user_lock);
3116 KMP_FATAL(LockIsUninitialized, func);
3120 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3121 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3123 return *((kmp_indirect_lock_t **)user_lock);
3128 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
3129 kmp_dyna_lockseq_t seq) {
3130 #if KMP_USE_ADAPTIVE_LOCKS 3131 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3132 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3133 seq = lockseq_queuing;
3137 if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
3138 seq = lockseq_queuing;
3141 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3142 kmp_indirect_lock_t *l =
3143 __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3144 KMP_I_LOCK_FUNC(l, init)(l->lock);
3146 20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n",
3150 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
3151 kmp_uint32 gtid = __kmp_entry_gtid();
3152 kmp_indirect_lock_t *l =
3153 __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3154 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3155 kmp_indirect_locktag_t tag = l->type;
3157 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3160 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3161 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3162 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3164 __kmp_indirect_lock_pool[tag] = l;
3166 __kmp_release_lock(&__kmp_global_lock, gtid);
3169 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3170 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3171 return KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3174 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3175 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3176 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3179 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3180 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3181 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3184 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3186 kmp_indirect_lock_t *l =
3187 __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3188 return KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3191 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3193 kmp_indirect_lock_t *l =
3194 __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3195 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3198 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3200 kmp_indirect_lock_t *l =
3201 __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3202 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3205 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3208 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3211 case lockseq_nested_tas:
3212 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3215 case lockseq_nested_futex:
3216 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3218 case lockseq_ticket:
3219 case lockseq_nested_ticket:
3220 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3221 case lockseq_queuing:
3222 case lockseq_nested_queuing:
3223 #if KMP_USE_ADAPTIVE_LOCKS 3224 case lockseq_adaptive:
3226 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3228 case lockseq_nested_drdpa:
3229 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3236 void __kmp_init_dynamic_user_locks() {
3238 if (__kmp_env_consistency_check) {
3239 __kmp_direct_set = direct_set_check;
3240 __kmp_direct_unset = direct_unset_check;
3241 __kmp_direct_test = direct_test_check;
3242 __kmp_indirect_set = indirect_set_check;
3243 __kmp_indirect_unset = indirect_unset_check;
3244 __kmp_indirect_test = indirect_test_check;
3246 __kmp_direct_set = direct_set;
3247 __kmp_direct_unset = direct_unset;
3248 __kmp_direct_test = direct_test;
3249 __kmp_indirect_set = indirect_set;
3250 __kmp_indirect_unset = indirect_unset;
3251 __kmp_indirect_test = indirect_test;
3256 if (__kmp_init_user_locks)
3260 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3261 __kmp_i_lock_table.table =
3262 (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3263 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)__kmp_allocate(
3264 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3265 __kmp_i_lock_table.next = 0;
3268 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3269 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3270 #if KMP_USE_ADAPTIVE_LOCKS 3271 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3273 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3275 __kmp_indirect_lock_size[locktag_rtm] =
sizeof(kmp_queuing_lock_t);
3277 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3279 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3281 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3282 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3283 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3286 #define fill_jumps(table, expand, sep) \ 3288 table[locktag##sep##ticket] = expand(ticket); \ 3289 table[locktag##sep##queuing] = expand(queuing); \ 3290 table[locktag##sep##drdpa] = expand(drdpa); \ 3293 #if KMP_USE_ADAPTIVE_LOCKS 3294 #define fill_table(table, expand) \ 3296 fill_jumps(table, expand, _); \ 3297 table[locktag_adaptive] = expand(queuing); \ 3298 fill_jumps(table, expand, _nested_); \ 3301 #define fill_table(table, expand) \ 3303 fill_jumps(table, expand, _); \ 3304 fill_jumps(table, expand, _nested_); \ 3306 #endif // KMP_USE_ADAPTIVE_LOCKS 3309 (void (*)(kmp_user_lock_p, const ident_t *)) __kmp_set_##l##_lock_location 3310 fill_table(__kmp_indirect_set_location, expand);
3313 (void (*)(kmp_user_lock_p, kmp_lock_flags_t)) __kmp_set_##l##_lock_flags 3314 fill_table(__kmp_indirect_set_flags, expand);
3317 (const ident_t *(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_location 3318 fill_table(__kmp_indirect_get_location, expand);
3321 (kmp_lock_flags_t(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_flags 3322 fill_table(__kmp_indirect_get_flags, expand);
3325 __kmp_init_user_locks = TRUE;
3329 void __kmp_cleanup_indirect_user_locks() {
3335 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3336 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3338 kmp_indirect_lock_t *ll = l;
3339 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3340 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n",
3342 __kmp_free(ll->lock);
3345 __kmp_indirect_lock_pool[k] = NULL;
3348 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3349 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3350 if (l->lock != NULL) {
3352 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3355 (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n",
3357 __kmp_free(l->lock);
3361 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3362 __kmp_free(__kmp_i_lock_table.table[i]);
3363 __kmp_free(__kmp_i_lock_table.table);
3365 __kmp_init_user_locks = FALSE;
3368 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3369 int __kmp_num_locks_in_block = 1;
3371 #else // KMP_USE_DYNAMIC_LOCK 3377 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3379 size_t __kmp_base_user_lock_size = 0;
3380 size_t __kmp_user_lock_size = 0;
3382 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3383 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3384 kmp_int32 gtid) = NULL;
3386 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3387 kmp_int32 gtid) = NULL;
3388 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3389 kmp_int32 gtid) = NULL;
3390 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3391 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3392 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3393 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3394 kmp_int32 gtid) = NULL;
3396 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3397 kmp_int32 gtid) = NULL;
3398 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3399 kmp_int32 gtid) = NULL;
3400 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3401 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3403 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3404 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3405 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3407 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3408 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3409 kmp_lock_flags_t flags) = NULL;
3411 void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind) {
3412 switch (user_lock_kind) {
3418 __kmp_base_user_lock_size =
sizeof(kmp_base_tas_lock_t);
3419 __kmp_user_lock_size =
sizeof(kmp_tas_lock_t);
3421 __kmp_get_user_lock_owner_ =
3422 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_tas_lock_owner);
3424 if (__kmp_env_consistency_check) {
3425 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3426 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3428 KMP_BIND_USER_LOCK(tas);
3429 KMP_BIND_NESTED_USER_LOCK(tas);
3432 __kmp_destroy_user_lock_ =
3433 (void (*)(kmp_user_lock_p))(&__kmp_destroy_tas_lock);
3435 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3437 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3439 __kmp_set_user_lock_location_ =
3440 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3442 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3444 __kmp_set_user_lock_flags_ =
3445 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3451 __kmp_base_user_lock_size =
sizeof(kmp_base_futex_lock_t);
3452 __kmp_user_lock_size =
sizeof(kmp_futex_lock_t);
3454 __kmp_get_user_lock_owner_ =
3455 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_futex_lock_owner);
3457 if (__kmp_env_consistency_check) {
3458 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3459 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3461 KMP_BIND_USER_LOCK(futex);
3462 KMP_BIND_NESTED_USER_LOCK(futex);
3465 __kmp_destroy_user_lock_ =
3466 (void (*)(kmp_user_lock_p))(&__kmp_destroy_futex_lock);
3468 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3470 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3472 __kmp_set_user_lock_location_ =
3473 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3475 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3477 __kmp_set_user_lock_flags_ =
3478 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3481 #endif // KMP_USE_FUTEX 3484 __kmp_base_user_lock_size =
sizeof(kmp_base_ticket_lock_t);
3485 __kmp_user_lock_size =
sizeof(kmp_ticket_lock_t);
3487 __kmp_get_user_lock_owner_ =
3488 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_owner);
3490 if (__kmp_env_consistency_check) {
3491 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3492 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3494 KMP_BIND_USER_LOCK(ticket);
3495 KMP_BIND_NESTED_USER_LOCK(ticket);
3498 __kmp_destroy_user_lock_ =
3499 (void (*)(kmp_user_lock_p))(&__kmp_destroy_ticket_lock);
3501 __kmp_is_user_lock_initialized_ =
3502 (int (*)(kmp_user_lock_p))(&__kmp_is_ticket_lock_initialized);
3504 __kmp_get_user_lock_location_ =
3505 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_location);
3507 __kmp_set_user_lock_location_ = (void (*)(
3508 kmp_user_lock_p,
const ident_t *))(&__kmp_set_ticket_lock_location);
3510 __kmp_get_user_lock_flags_ =
3511 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_flags);
3513 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3514 &__kmp_set_ticket_lock_flags);
3518 __kmp_base_user_lock_size =
sizeof(kmp_base_queuing_lock_t);
3519 __kmp_user_lock_size =
sizeof(kmp_queuing_lock_t);
3521 __kmp_get_user_lock_owner_ =
3522 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3524 if (__kmp_env_consistency_check) {
3525 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3526 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3528 KMP_BIND_USER_LOCK(queuing);
3529 KMP_BIND_NESTED_USER_LOCK(queuing);
3532 __kmp_destroy_user_lock_ =
3533 (void (*)(kmp_user_lock_p))(&__kmp_destroy_queuing_lock);
3535 __kmp_is_user_lock_initialized_ =
3536 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3538 __kmp_get_user_lock_location_ =
3539 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3541 __kmp_set_user_lock_location_ = (void (*)(
3542 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3544 __kmp_get_user_lock_flags_ =
3545 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3547 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3548 &__kmp_set_queuing_lock_flags);
3551 #if KMP_USE_ADAPTIVE_LOCKS 3553 __kmp_base_user_lock_size =
sizeof(kmp_base_adaptive_lock_t);
3554 __kmp_user_lock_size =
sizeof(kmp_adaptive_lock_t);
3556 __kmp_get_user_lock_owner_ =
3557 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3559 if (__kmp_env_consistency_check) {
3560 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3562 KMP_BIND_USER_LOCK(adaptive);
3565 __kmp_destroy_user_lock_ =
3566 (void (*)(kmp_user_lock_p))(&__kmp_destroy_adaptive_lock);
3568 __kmp_is_user_lock_initialized_ =
3569 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3571 __kmp_get_user_lock_location_ =
3572 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3574 __kmp_set_user_lock_location_ = (void (*)(
3575 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3577 __kmp_get_user_lock_flags_ =
3578 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3580 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3581 &__kmp_set_queuing_lock_flags);
3584 #endif // KMP_USE_ADAPTIVE_LOCKS 3587 __kmp_base_user_lock_size =
sizeof(kmp_base_drdpa_lock_t);
3588 __kmp_user_lock_size =
sizeof(kmp_drdpa_lock_t);
3590 __kmp_get_user_lock_owner_ =
3591 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_owner);
3593 if (__kmp_env_consistency_check) {
3594 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3595 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3597 KMP_BIND_USER_LOCK(drdpa);
3598 KMP_BIND_NESTED_USER_LOCK(drdpa);
3601 __kmp_destroy_user_lock_ =
3602 (void (*)(kmp_user_lock_p))(&__kmp_destroy_drdpa_lock);
3604 __kmp_is_user_lock_initialized_ =
3605 (int (*)(kmp_user_lock_p))(&__kmp_is_drdpa_lock_initialized);
3607 __kmp_get_user_lock_location_ =
3608 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_location);
3610 __kmp_set_user_lock_location_ = (void (*)(
3611 kmp_user_lock_p,
const ident_t *))(&__kmp_set_drdpa_lock_location);
3613 __kmp_get_user_lock_flags_ =
3614 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_flags);
3616 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3617 &__kmp_set_drdpa_lock_flags);
3625 kmp_lock_table_t __kmp_user_lock_table = {1, 0, NULL};
3626 kmp_user_lock_p __kmp_lock_pool = NULL;
3629 kmp_block_of_locks *__kmp_lock_blocks = NULL;
3630 int __kmp_num_locks_in_block = 1;
3632 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3634 kmp_lock_index_t index;
3635 if (__kmp_user_lock_table.used >= __kmp_user_lock_table.allocated) {
3636 kmp_lock_index_t size;
3637 kmp_user_lock_p *table;
3639 if (__kmp_user_lock_table.allocated == 0) {
3642 size = __kmp_user_lock_table.allocated * 2;
3644 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof(kmp_user_lock_p) * size);
3645 KMP_MEMCPY(table + 1, __kmp_user_lock_table.table + 1,
3646 sizeof(kmp_user_lock_p) * (__kmp_user_lock_table.used - 1));
3647 table[0] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3652 __kmp_user_lock_table.table = table;
3653 __kmp_user_lock_table.allocated = size;
3655 KMP_DEBUG_ASSERT(__kmp_user_lock_table.used <
3656 __kmp_user_lock_table.allocated);
3657 index = __kmp_user_lock_table.used;
3658 __kmp_user_lock_table.table[index] = lck;
3659 ++__kmp_user_lock_table.used;
3663 static kmp_user_lock_p __kmp_lock_block_allocate() {
3665 static int last_index = 0;
3666 if ((last_index >= __kmp_num_locks_in_block) || (__kmp_lock_blocks == NULL)) {
3670 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3671 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3673 (
char *)__kmp_allocate(space_for_locks +
sizeof(kmp_block_of_locks));
3675 kmp_block_of_locks *new_block =
3676 (kmp_block_of_locks *)(&buffer[space_for_locks]);
3677 new_block->next_block = __kmp_lock_blocks;
3678 new_block->locks = (
void *)buffer;
3681 __kmp_lock_blocks = new_block;
3683 kmp_user_lock_p ret = (kmp_user_lock_p)(&(
3684 ((
char *)(__kmp_lock_blocks->locks))[last_index * __kmp_user_lock_size]));
3691 kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3692 kmp_lock_flags_t flags) {
3693 kmp_user_lock_p lck;
3694 kmp_lock_index_t index;
3695 KMP_DEBUG_ASSERT(user_lock);
3697 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3699 if (__kmp_lock_pool == NULL) {
3704 ANNOTATE_IGNORE_WRITES_BEGIN();
3705 if (__kmp_num_locks_in_block <= 1) {
3706 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3708 lck = __kmp_lock_block_allocate();
3710 ANNOTATE_IGNORE_WRITES_END();
3714 index = __kmp_lock_table_insert(lck);
3717 lck = __kmp_lock_pool;
3718 index = __kmp_lock_pool->pool.index;
3719 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3724 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3725 *((kmp_lock_index_t *)user_lock) = index;
3727 *((kmp_user_lock_p *)user_lock) = lck;
3731 __kmp_set_user_lock_flags(lck, flags);
3733 __kmp_release_lock(&__kmp_global_lock, gtid);
3739 void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
3740 kmp_user_lock_p lck) {
3741 KMP_DEBUG_ASSERT(user_lock != NULL);
3742 KMP_DEBUG_ASSERT(lck != NULL);
3744 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3746 lck->pool.next = __kmp_lock_pool;
3747 __kmp_lock_pool = lck;
3748 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3749 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3750 KMP_DEBUG_ASSERT(0 < index && index <= __kmp_user_lock_table.used);
3751 lck->pool.index = index;
3754 __kmp_release_lock(&__kmp_global_lock, gtid);
3757 kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func) {
3758 kmp_user_lock_p lck = NULL;
3760 if (__kmp_env_consistency_check) {
3761 if (user_lock == NULL) {
3762 KMP_FATAL(LockIsUninitialized, func);
3766 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3767 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3768 if (__kmp_env_consistency_check) {
3769 if (!(0 < index && index < __kmp_user_lock_table.used)) {
3770 KMP_FATAL(LockIsUninitialized, func);
3773 KMP_DEBUG_ASSERT(0 < index && index < __kmp_user_lock_table.used);
3774 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3775 lck = __kmp_user_lock_table.table[index];
3777 lck = *((kmp_user_lock_p *)user_lock);
3780 if (__kmp_env_consistency_check) {
3782 KMP_FATAL(LockIsUninitialized, func);
3789 void __kmp_cleanup_user_locks(
void) {
3792 __kmp_lock_pool = NULL;
3794 #define IS_CRITICAL(lck) \ 3795 ((__kmp_get_user_lock_flags_ != NULL) && \ 3796 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section)) 3821 while (__kmp_user_lock_table.used > 1) {
3826 kmp_user_lock_p lck =
3827 __kmp_user_lock_table.table[--__kmp_user_lock_table.used];
3829 if ((__kmp_is_user_lock_initialized_ != NULL) &&
3830 (*__kmp_is_user_lock_initialized_)(lck)) {
3834 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3835 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
3837 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->
psource, 0);
3838 KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
3839 __kmp_str_loc_free(&str_loc);
3843 if (IS_CRITICAL(lck)) {
3846 (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n",
3847 lck, *(
void **)lck));
3849 KA_TRACE(20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
3855 __kmp_destroy_user_lock(lck);
3859 if (__kmp_lock_blocks == NULL) {
3867 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
3868 __kmp_user_lock_table.table = NULL;
3869 __kmp_user_lock_table.allocated = 0;
3871 while (table_ptr != NULL) {
3874 kmp_user_lock_p *next = (kmp_user_lock_p *)(table_ptr[0]);
3875 __kmp_free(table_ptr);
3880 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
3881 __kmp_lock_blocks = NULL;
3883 while (block_ptr != NULL) {
3884 kmp_block_of_locks_t *next = block_ptr->next_block;
3885 __kmp_free(block_ptr->locks);
3890 TCW_4(__kmp_init_user_locks, FALSE);
3893 #endif // KMP_USE_DYNAMIC_LOCK