18 #define USE_CHECKS_COMMON 20 #define KMP_INLINE_SUBR 1 22 void kmp_threadprivate_insert_private_data(
int gtid,
void *pc_addr,
23 void *data_addr,
size_t pc_size);
24 struct private_common *kmp_threadprivate_insert(
int gtid,
void *pc_addr,
28 struct shared_table __kmp_threadprivate_d_table;
31 #ifdef KMP_INLINE_SUBR 34 struct private_common *
35 __kmp_threadprivate_find_task_common(
struct common_table *tbl,
int gtid,
39 struct private_common *tn;
41 #ifdef KMP_TASK_COMMON_DEBUG 42 KC_TRACE(10, (
"__kmp_threadprivate_find_task_common: thread#%d, called with " 48 for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) {
49 if (tn->gbl_addr == pc_addr) {
50 #ifdef KMP_TASK_COMMON_DEBUG 51 KC_TRACE(10, (
"__kmp_threadprivate_find_task_common: thread#%d, found " 62 #ifdef KMP_INLINE_SUBR 65 struct shared_common *
66 __kmp_find_shared_task_common(
struct shared_table *tbl,
int gtid,
68 struct shared_common *tn;
70 for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) {
71 if (tn->gbl_addr == pc_addr) {
72 #ifdef KMP_TASK_COMMON_DEBUG 75 (
"__kmp_find_shared_task_common: thread#%d, found node %p on list\n",
86 static struct private_data *__kmp_init_common_data(
void *pc_addr,
88 struct private_data *d;
92 d = (
struct private_data *)__kmp_allocate(
sizeof(
struct private_data));
103 for (i = pc_size; i > 0; --i) {
105 d->data = __kmp_allocate(pc_size);
106 KMP_MEMCPY(d->data, pc_addr, pc_size);
115 static void __kmp_copy_common_data(
void *pc_addr,
struct private_data *d) {
116 char *addr = (
char *)pc_addr;
119 for (offset = 0; d != 0; d = d->next) {
120 for (i = d->more; i > 0; --i) {
122 memset(&addr[offset],
'\0', d->size);
124 KMP_MEMCPY(&addr[offset], d->data, d->size);
131 void __kmp_common_initialize(
void) {
132 if (!TCR_4(__kmp_init_common)) {
138 __kmp_threadpriv_cache_list = NULL;
142 for (gtid = 0; gtid < __kmp_threads_capacity; gtid++)
143 if (__kmp_root[gtid]) {
144 KMP_DEBUG_ASSERT(__kmp_root[gtid]->r.r_uber_thread);
145 for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q)
147 !__kmp_root[gtid]->r.r_uber_thread->th.th_pri_common->data[q]);
153 for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q)
154 __kmp_threadprivate_d_table.data[q] = 0;
156 TCW_4(__kmp_init_common, TRUE);
162 void __kmp_common_destroy(
void) {
163 if (TCR_4(__kmp_init_common)) {
166 TCW_4(__kmp_init_common, FALSE);
168 for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) {
170 struct private_common *tn;
171 struct shared_common *d_tn;
177 for (d_tn = __kmp_threadprivate_d_table.data[q]; d_tn;
180 if (d_tn->dt.dtorv != 0) {
181 for (gtid = 0; gtid < __kmp_all_nth; ++gtid) {
182 if (__kmp_threads[gtid]) {
183 if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid))
184 : (!KMP_UBER_GTID(gtid))) {
185 tn = __kmp_threadprivate_find_task_common(
186 __kmp_threads[gtid]->th.th_pri_common, gtid,
189 (*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len);
194 if (d_tn->obj_init != 0) {
195 (*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len);
199 if (d_tn->dt.dtor != 0) {
200 for (gtid = 0; gtid < __kmp_all_nth; ++gtid) {
201 if (__kmp_threads[gtid]) {
202 if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid))
203 : (!KMP_UBER_GTID(gtid))) {
204 tn = __kmp_threadprivate_find_task_common(
205 __kmp_threads[gtid]->th.th_pri_common, gtid,
208 (*d_tn->dt.dtor)(tn->par_addr);
213 if (d_tn->obj_init != 0) {
214 (*d_tn->dt.dtor)(d_tn->obj_init);
219 __kmp_threadprivate_d_table.data[q] = 0;
225 void __kmp_common_destroy_gtid(
int gtid) {
226 struct private_common *tn;
227 struct shared_common *d_tn;
229 if (!TCR_4(__kmp_init_gtid)) {
236 KC_TRACE(10, (
"__kmp_common_destroy_gtid: T#%d called\n", gtid));
237 if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) : (!KMP_UBER_GTID(gtid))) {
239 if (TCR_4(__kmp_init_common)) {
244 for (tn = __kmp_threads[gtid]->th.th_pri_head; tn; tn = tn->link) {
246 d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid,
249 KMP_DEBUG_ASSERT(d_tn);
252 if (d_tn->dt.dtorv != 0) {
253 (void)(*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len);
255 if (d_tn->obj_init != 0) {
256 (void)(*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len);
259 if (d_tn->dt.dtor != 0) {
260 (void)(*d_tn->dt.dtor)(tn->par_addr);
262 if (d_tn->obj_init != 0) {
263 (void)(*d_tn->dt.dtor)(d_tn->obj_init);
267 KC_TRACE(30, (
"__kmp_common_destroy_gtid: T#%d threadprivate destructors " 274 #ifdef KMP_TASK_COMMON_DEBUG 275 static void dump_list(
void) {
278 for (p = 0; p < __kmp_all_nth; ++p) {
279 if (!__kmp_threads[p])
281 for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) {
282 if (__kmp_threads[p]->th.th_pri_common->data[q]) {
283 struct private_common *tn;
285 KC_TRACE(10, (
"\tdump_list: gtid:%d addresses\n", p));
287 for (tn = __kmp_threads[p]->th.th_pri_common->data[q]; tn;
290 (
"\tdump_list: THREADPRIVATE: Serial %p -> Parallel %p\n",
291 tn->gbl_addr, tn->par_addr));
300 void kmp_threadprivate_insert_private_data(
int gtid,
void *pc_addr,
301 void *data_addr,
size_t pc_size) {
302 struct shared_common **lnk_tn, *d_tn;
303 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
304 __kmp_threads[gtid]->th.th_root->r.r_active == 0);
306 d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid,
310 d_tn = (
struct shared_common *)__kmp_allocate(
sizeof(
struct shared_common));
312 d_tn->gbl_addr = pc_addr;
313 d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size);
323 d_tn->cmn_size = pc_size;
325 __kmp_acquire_lock(&__kmp_global_lock, gtid);
327 lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]);
329 d_tn->next = *lnk_tn;
332 __kmp_release_lock(&__kmp_global_lock, gtid);
336 struct private_common *kmp_threadprivate_insert(
int gtid,
void *pc_addr,
339 struct private_common *tn, **tt;
340 struct shared_common *d_tn;
343 __kmp_acquire_lock(&__kmp_global_lock, gtid);
345 tn = (
struct private_common *)__kmp_allocate(
sizeof(
struct private_common));
347 tn->gbl_addr = pc_addr;
349 d_tn = __kmp_find_shared_task_common(
350 &__kmp_threadprivate_d_table, gtid,
356 if (d_tn->pod_init == 0 && d_tn->obj_init == 0) {
357 d_tn->cmn_size = pc_size;
360 if (d_tn->ct.ctorv != 0) {
363 }
else if (d_tn->cct.cctorv != 0) {
366 d_tn->obj_init = (
void *)__kmp_allocate(d_tn->cmn_size);
367 (void)(*d_tn->cct.cctorv)(d_tn->obj_init, pc_addr, d_tn->vec_len);
369 d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size);
372 if (d_tn->ct.ctor != 0) {
375 }
else if (d_tn->cct.cctor != 0) {
378 d_tn->obj_init = (
void *)__kmp_allocate(d_tn->cmn_size);
379 (void)(*d_tn->cct.cctor)(d_tn->obj_init, pc_addr);
381 d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size);
386 struct shared_common **lnk_tn;
388 d_tn = (
struct shared_common *)__kmp_allocate(
sizeof(
struct shared_common));
389 d_tn->gbl_addr = pc_addr;
390 d_tn->cmn_size = pc_size;
391 d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size);
401 lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]);
403 d_tn->next = *lnk_tn;
407 tn->cmn_size = d_tn->cmn_size;
409 if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid))) {
410 tn->par_addr = (
void *)pc_addr;
412 tn->par_addr = (
void *)__kmp_allocate(tn->cmn_size);
415 __kmp_release_lock(&__kmp_global_lock, gtid);
418 #ifdef USE_CHECKS_COMMON 419 if (pc_size > d_tn->cmn_size) {
421 10, (
"__kmp_threadprivate_insert: THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC
422 " ,%" KMP_UINTPTR_SPEC
")\n",
423 pc_addr, pc_size, d_tn->cmn_size));
424 KMP_FATAL(TPCommonBlocksInconsist);
428 tt = &(__kmp_threads[gtid]->th.th_pri_common->data[KMP_HASH(pc_addr)]);
430 #ifdef KMP_TASK_COMMON_DEBUG 434 (
"__kmp_threadprivate_insert: WARNING! thread#%d: collision on %p\n",
441 #ifdef KMP_TASK_COMMON_DEBUG 443 (
"__kmp_threadprivate_insert: thread#%d, inserted node %p on list\n",
450 tn->link = __kmp_threads[gtid]->th.th_pri_head;
451 __kmp_threads[gtid]->th.th_pri_head = tn;
453 if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid)))
467 if (d_tn->ct.ctorv != 0) {
468 (void)(*d_tn->ct.ctorv)(tn->par_addr, d_tn->vec_len);
469 }
else if (d_tn->cct.cctorv != 0) {
470 (void)(*d_tn->cct.cctorv)(tn->par_addr, d_tn->obj_init, d_tn->vec_len);
471 }
else if (tn->par_addr != tn->gbl_addr) {
472 __kmp_copy_common_data(tn->par_addr, d_tn->pod_init);
475 if (d_tn->ct.ctor != 0) {
476 (void)(*d_tn->ct.ctor)(tn->par_addr);
477 }
else if (d_tn->cct.cctor != 0) {
478 (void)(*d_tn->cct.cctor)(tn->par_addr, d_tn->obj_init);
479 }
else if (tn->par_addr != tn->gbl_addr) {
480 __kmp_copy_common_data(tn->par_addr, d_tn->pod_init);
508 struct shared_common *d_tn, **lnk_tn;
510 KC_TRACE(10, (
"__kmpc_threadprivate_register: called\n"));
512 #ifdef USE_CHECKS_COMMON 514 KMP_ASSERT(cctor == 0);
518 d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, -1, data);
521 d_tn = (
struct shared_common *)__kmp_allocate(
sizeof(
struct shared_common));
522 d_tn->gbl_addr = data;
524 d_tn->ct.ctor = ctor;
525 d_tn->cct.cctor = cctor;
526 d_tn->dt.dtor = dtor;
534 lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]);
536 d_tn->next = *lnk_tn;
541 void *__kmpc_threadprivate(
ident_t *loc, kmp_int32 global_tid,
void *data,
544 struct private_common *tn;
546 KC_TRACE(10, (
"__kmpc_threadprivate: T#%d called\n", global_tid));
548 #ifdef USE_CHECKS_COMMON 549 if (!__kmp_init_serial)
550 KMP_FATAL(RTLNotInitialized);
553 if (!__kmp_threads[global_tid]->th.th_root->r.r_active && !__kmp_foreign_tp) {
558 KC_TRACE(20, (
"__kmpc_threadprivate: T#%d inserting private data\n",
560 kmp_threadprivate_insert_private_data(global_tid, data, data, size);
566 (
"__kmpc_threadprivate: T#%d try to find private data at address %p\n",
568 tn = __kmp_threadprivate_find_task_common(
569 __kmp_threads[global_tid]->th.th_pri_common, global_tid, data);
572 KC_TRACE(20, (
"__kmpc_threadprivate: T#%d found data\n", global_tid));
573 #ifdef USE_CHECKS_COMMON 574 if ((
size_t)size > tn->cmn_size) {
575 KC_TRACE(10, (
"THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC
576 " ,%" KMP_UINTPTR_SPEC
")\n",
577 data, size, tn->cmn_size));
578 KMP_FATAL(TPCommonBlocksInconsist);
585 KC_TRACE(20, (
"__kmpc_threadprivate: T#%d inserting data\n", global_tid));
586 tn = kmp_threadprivate_insert(global_tid, data, data, size);
591 KC_TRACE(10, (
"__kmpc_threadprivate: T#%d exiting; return value = %p\n",
597 static kmp_cached_addr_t *__kmp_find_cache(
void *data) {
598 kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list;
599 while (ptr && ptr->data != data)
617 kmp_int32 global_tid,
621 KC_TRACE(10, (
"__kmpc_threadprivate_cached: T#%d called with cache: %p, " 622 "address: %p, size: %" KMP_SIZE_T_SPEC
"\n",
623 global_tid, *cache, data, size));
625 if (TCR_PTR(*cache) == 0) {
626 __kmp_acquire_lock(&__kmp_global_lock, global_tid);
628 if (TCR_PTR(*cache) == 0) {
629 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
632 kmp_cached_addr_t *tp_cache_addr;
634 tp_cache_addr = __kmp_find_cache(data);
635 if (!tp_cache_addr) {
637 KMP_ITT_IGNORE(my_cache = (
void **)__kmp_allocate(
638 sizeof(
void *) * __kmp_tp_capacity +
639 sizeof(kmp_cached_addr_t)););
641 KC_TRACE(50, (
"__kmpc_threadprivate_cached: T#%d allocated cache at " 643 global_tid, my_cache));
647 tp_cache_addr = (kmp_cached_addr_t *)&my_cache[__kmp_tp_capacity];
648 tp_cache_addr->addr = my_cache;
649 tp_cache_addr->data = data;
650 tp_cache_addr->compiler_cache = cache;
651 tp_cache_addr->next = __kmp_threadpriv_cache_list;
652 __kmp_threadpriv_cache_list = tp_cache_addr;
654 my_cache = tp_cache_addr->addr;
655 tp_cache_addr->compiler_cache = cache;
659 TCW_PTR(*cache, my_cache);
660 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
664 __kmp_release_lock(&__kmp_global_lock, global_tid);
668 if ((ret = TCR_PTR((*cache)[global_tid])) == 0) {
669 ret = __kmpc_threadprivate(loc, global_tid, data, (
size_t)size);
671 TCW_PTR((*cache)[global_tid], ret);
674 (
"__kmpc_threadprivate_cached: T#%d exiting; return value = %p\n",
681 void __kmp_threadprivate_resize_cache(
int newCapacity) {
682 KC_TRACE(10, (
"__kmp_threadprivate_resize_cache: called with size: %d\n",
685 kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list;
690 KMP_ITT_IGNORE(my_cache =
691 (
void **)__kmp_allocate(
sizeof(
void *) * newCapacity +
692 sizeof(kmp_cached_addr_t)););
694 KC_TRACE(50, (
"__kmp_threadprivate_resize_cache: allocated cache at %p\n",
697 void **old_cache = ptr->addr;
698 for (
int i = 0; i < __kmp_tp_capacity; ++i) {
699 my_cache[i] = old_cache[i];
703 kmp_cached_addr_t *tp_cache_addr;
704 tp_cache_addr = (kmp_cached_addr_t *)&my_cache[newCapacity];
705 tp_cache_addr->addr = my_cache;
706 tp_cache_addr->data = ptr->data;
707 tp_cache_addr->compiler_cache = ptr->compiler_cache;
708 tp_cache_addr->next = __kmp_threadpriv_cache_list;
709 __kmp_threadpriv_cache_list = tp_cache_addr;
720 (void)KMP_COMPARE_AND_STORE_PTR(tp_cache_addr->compiler_cache, old_cache,
735 *(
volatile int *)&__kmp_tp_capacity = newCapacity;
751 size_t vector_length) {
752 struct shared_common *d_tn, **lnk_tn;
754 KC_TRACE(10, (
"__kmpc_threadprivate_register_vec: called\n"));
756 #ifdef USE_CHECKS_COMMON 758 KMP_ASSERT(cctor == 0);
761 d_tn = __kmp_find_shared_task_common(
762 &__kmp_threadprivate_d_table, -1,
766 d_tn = (
struct shared_common *)__kmp_allocate(
sizeof(
struct shared_common));
767 d_tn->gbl_addr = data;
769 d_tn->ct.ctorv = ctor;
770 d_tn->cct.cctorv = cctor;
771 d_tn->dt.dtorv = dtor;
773 d_tn->vec_len = (size_t)vector_length;
776 lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]);
778 d_tn->next = *lnk_tn;
783 void __kmp_cleanup_threadprivate_caches() {
784 kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list;
787 void **cache = ptr->addr;
788 __kmp_threadpriv_cache_list = ptr->next;
789 if (*ptr->compiler_cache)
790 *ptr->compiler_cache = NULL;
791 ptr->compiler_cache = NULL;
798 ptr = __kmp_threadpriv_cache_list;
void(* kmpc_dtor)(void *)
void(* kmpc_dtor_vec)(void *, size_t)
void *(* kmpc_ctor_vec)(void *, size_t)
void * __kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid, void *data, size_t size, void ***cache)
void *(* kmpc_cctor_vec)(void *, void *, size_t)
void *(* kmpc_cctor)(void *, void *)
void __kmpc_threadprivate_register(ident_t *loc, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor)
void *(* kmpc_ctor)(void *)
void __kmpc_threadprivate_register_vec(ident_t *loc, void *data, kmpc_ctor_vec ctor, kmpc_cctor_vec cctor, kmpc_dtor_vec dtor, size_t vector_length)