16 #include "kmp_atomic.h" 19 typedef unsigned char uchar;
20 typedef unsigned short ushort;
555 #ifndef KMP_GOMP_COMPAT 556 int __kmp_atomic_mode = 1;
558 int __kmp_atomic_mode = 2;
563 kmp_atomic_lock_t __kmp_atomic_lock;
564 kmp_atomic_lock_t __kmp_atomic_lock_1i;
565 kmp_atomic_lock_t __kmp_atomic_lock_2i;
566 kmp_atomic_lock_t __kmp_atomic_lock_4i;
567 kmp_atomic_lock_t __kmp_atomic_lock_4r;
568 kmp_atomic_lock_t __kmp_atomic_lock_8i;
569 kmp_atomic_lock_t __kmp_atomic_lock_8r;
570 kmp_atomic_lock_t __kmp_atomic_lock_8c;
571 kmp_atomic_lock_t __kmp_atomic_lock_10r;
572 kmp_atomic_lock_t __kmp_atomic_lock_16r;
573 kmp_atomic_lock_t __kmp_atomic_lock_16c;
574 kmp_atomic_lock_t __kmp_atomic_lock_20c;
575 kmp_atomic_lock_t __kmp_atomic_lock_32c;
585 #define KMP_ATOMIC_VOLATILE volatile 587 #if ( KMP_ARCH_X86 ) && KMP_HAVE_QUAD 589 static inline void operator +=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q += rhs.q; };
590 static inline void operator -=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q -= rhs.q; };
591 static inline void operator *=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q *= rhs.q; };
592 static inline void operator /=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q /= rhs.q; };
593 static inline bool operator < ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q < rhs.q; }
594 static inline bool operator > ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q > rhs.q; }
596 static inline void operator +=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q += rhs.q; };
597 static inline void operator -=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q -= rhs.q; };
598 static inline void operator *=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q *= rhs.q; };
599 static inline void operator /=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q /= rhs.q; };
600 static inline bool operator < ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q < rhs.q; }
601 static inline bool operator > ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q > rhs.q; }
603 static inline void operator +=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q += rhs.q; };
604 static inline void operator -=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q -= rhs.q; };
605 static inline void operator *=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q *= rhs.q; };
606 static inline void operator /=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q /= rhs.q; };
608 static inline void operator +=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q += rhs.q; };
609 static inline void operator -=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q -= rhs.q; };
610 static inline void operator *=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q *= rhs.q; };
611 static inline void operator /=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q /= rhs.q; };
624 #define KMP_CHECK_GTID \ 625 if ( gtid == KMP_GTID_UNKNOWN ) { \ 626 gtid = __kmp_entry_gtid(); \ 627 } // check and get gtid when needed 633 #define ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE, RET_TYPE) \ 634 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \ 636 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 637 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 641 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat 642 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char 643 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short 644 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int 645 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float 646 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int 647 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double 648 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex 649 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double 650 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad 651 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex 652 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex 653 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex 661 #define OP_CRITICAL(OP,LCK_ID) \ 662 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 666 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 691 #ifdef KMP_GOMP_COMPAT 692 # define OP_GOMP_CRITICAL(OP,FLAG) \ 693 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 695 OP_CRITICAL( OP, 0 ); \ 699 # define OP_GOMP_CRITICAL(OP,FLAG) 703 # define KMP_DO_PAUSE _mm_delay_32( 1 ) 705 # define KMP_DO_PAUSE KMP_CPU_PAUSE() 713 #define OP_CMPXCHG(TYPE,BITS,OP) \ 715 TYPE old_value, new_value; \ 716 old_value = *(TYPE volatile *)lhs; \ 717 new_value = old_value OP rhs; \ 718 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 719 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 720 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 724 old_value = *(TYPE volatile *)lhs; \ 725 new_value = old_value OP rhs; \ 737 #define OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \ 741 kmp_int##BITS *vvv; \ 743 struct _sss old_value, new_value; \ 744 old_value.vvv = ( kmp_int##BITS * )&old_value.cmp; \ 745 new_value.vvv = ( kmp_int##BITS * )&new_value.cmp; \ 746 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \ 747 new_value.cmp = old_value.cmp OP rhs; \ 748 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 749 *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ 750 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv ) ) \ 754 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \ 755 new_value.cmp = old_value.cmp OP rhs; \ 759 #endif // USE_CMPXCHG_FIX 761 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 765 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 766 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 767 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 769 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \ 772 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 773 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 774 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 775 OP_CMPXCHG(TYPE,BITS,OP) \ 780 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 781 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 782 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 783 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \ 791 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 792 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 793 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 794 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 796 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \ 799 OP_CRITICAL(OP##=,LCK_ID) \ 803 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 804 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 805 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 806 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 807 OP_CMPXCHG(TYPE,BITS,OP) \ 810 OP_CRITICAL(OP##=,LCK_ID) \ 816 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 817 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 818 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 819 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 820 OP_CMPXCHG(TYPE,BITS,OP) \ 823 OP_CRITICAL(OP##=,LCK_ID) \ 827 #endif // USE_CMPXCHG_FIX 831 ATOMIC_FIXED_ADD( fixed4, add, kmp_int32, 32, +, 4i, 3, 0 )
832 ATOMIC_FIXED_ADD( fixed4, sub, kmp_int32, 32, -, 4i, 3, 0 )
834 ATOMIC_CMPXCHG( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
835 ATOMIC_CMPXCHG( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
838 ATOMIC_FIXED_ADD( fixed8, add, kmp_int64, 64, +, 8i, 7, KMP_ARCH_X86 )
839 ATOMIC_FIXED_ADD( fixed8, sub, kmp_int64, 64, -, 8i, 7, KMP_ARCH_X86 )
841 ATOMIC_CMPXCHG( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
842 ATOMIC_CMPXCHG( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
859 ATOMIC_CMPXCHG( fixed1, add, kmp_int8, 8, +, 1i, 0, KMP_ARCH_X86 )
860 ATOMIC_CMPXCHG( fixed1, andb, kmp_int8, 8, &, 1i, 0, 0 )
861 ATOMIC_CMPXCHG( fixed1, div, kmp_int8, 8, /, 1i, 0, KMP_ARCH_X86 )
862 ATOMIC_CMPXCHG( fixed1u, div, kmp_uint8, 8, /, 1i, 0, KMP_ARCH_X86 )
863 ATOMIC_CMPXCHG( fixed1, mul, kmp_int8, 8, *, 1i, 0, KMP_ARCH_X86 )
864 ATOMIC_CMPXCHG( fixed1, orb, kmp_int8, 8, |, 1i, 0, 0 )
865 ATOMIC_CMPXCHG( fixed1, shl, kmp_int8, 8, <<, 1i, 0, KMP_ARCH_X86 )
866 ATOMIC_CMPXCHG( fixed1, shr, kmp_int8, 8, >>, 1i, 0, KMP_ARCH_X86 )
867 ATOMIC_CMPXCHG( fixed1u, shr, kmp_uint8, 8, >>, 1i, 0, KMP_ARCH_X86 )
868 ATOMIC_CMPXCHG( fixed1, sub, kmp_int8, 8, -, 1i, 0, KMP_ARCH_X86 )
869 ATOMIC_CMPXCHG( fixed1, xor, kmp_int8, 8, ^, 1i, 0, 0 )
870 ATOMIC_CMPXCHG( fixed2, add, kmp_int16, 16, +, 2i, 1, KMP_ARCH_X86 )
871 ATOMIC_CMPXCHG( fixed2, andb, kmp_int16, 16, &, 2i, 1, 0 )
872 ATOMIC_CMPXCHG( fixed2, div, kmp_int16, 16, /, 2i, 1, KMP_ARCH_X86 )
873 ATOMIC_CMPXCHG( fixed2u, div, kmp_uint16, 16, /, 2i, 1, KMP_ARCH_X86 )
874 ATOMIC_CMPXCHG( fixed2, mul, kmp_int16, 16, *, 2i, 1, KMP_ARCH_X86 )
875 ATOMIC_CMPXCHG( fixed2, orb, kmp_int16, 16, |, 2i, 1, 0 )
876 ATOMIC_CMPXCHG( fixed2, shl, kmp_int16, 16, <<, 2i, 1, KMP_ARCH_X86 )
877 ATOMIC_CMPXCHG( fixed2, shr, kmp_int16, 16, >>, 2i, 1, KMP_ARCH_X86 )
878 ATOMIC_CMPXCHG( fixed2u, shr, kmp_uint16, 16, >>, 2i, 1, KMP_ARCH_X86 )
879 ATOMIC_CMPXCHG( fixed2, sub, kmp_int16, 16, -, 2i, 1, KMP_ARCH_X86 )
880 ATOMIC_CMPXCHG( fixed2, xor, kmp_int16, 16, ^, 2i, 1, 0 )
881 ATOMIC_CMPXCHG( fixed4, andb, kmp_int32, 32, &, 4i, 3, 0 )
882 ATOMIC_CMPXCHG( fixed4, div, kmp_int32, 32, /, 4i, 3, KMP_ARCH_X86 )
883 ATOMIC_CMPXCHG( fixed4u, div, kmp_uint32, 32, /, 4i, 3, KMP_ARCH_X86 )
884 ATOMIC_CMPXCHG( fixed4, mul, kmp_int32, 32, *, 4i, 3, KMP_ARCH_X86 )
885 ATOMIC_CMPXCHG( fixed4, orb, kmp_int32, 32, |, 4i, 3, 0 )
886 ATOMIC_CMPXCHG( fixed4, shl, kmp_int32, 32, <<, 4i, 3, KMP_ARCH_X86 )
887 ATOMIC_CMPXCHG( fixed4, shr, kmp_int32, 32, >>, 4i, 3, KMP_ARCH_X86 )
888 ATOMIC_CMPXCHG( fixed4u, shr, kmp_uint32, 32, >>, 4i, 3, KMP_ARCH_X86 )
889 ATOMIC_CMPXCHG( fixed4, xor, kmp_int32, 32, ^, 4i, 3, 0 )
890 ATOMIC_CMPXCHG( fixed8, andb, kmp_int64, 64, &, 8i, 7, KMP_ARCH_X86 )
891 ATOMIC_CMPXCHG( fixed8, div, kmp_int64, 64, /, 8i, 7, KMP_ARCH_X86 )
892 ATOMIC_CMPXCHG( fixed8u, div, kmp_uint64, 64, /, 8i, 7, KMP_ARCH_X86 )
893 ATOMIC_CMPXCHG( fixed8, mul, kmp_int64, 64, *, 8i, 7, KMP_ARCH_X86 )
894 ATOMIC_CMPXCHG( fixed8, orb, kmp_int64, 64, |, 8i, 7, KMP_ARCH_X86 )
895 ATOMIC_CMPXCHG( fixed8, shl, kmp_int64, 64, <<, 8i, 7, KMP_ARCH_X86 )
896 ATOMIC_CMPXCHG( fixed8, shr, kmp_int64, 64, >>, 8i, 7, KMP_ARCH_X86 )
897 ATOMIC_CMPXCHG( fixed8u, shr, kmp_uint64, 64, >>, 8i, 7, KMP_ARCH_X86 )
898 ATOMIC_CMPXCHG( fixed8, xor, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
899 ATOMIC_CMPXCHG( float4, div, kmp_real32, 32, /, 4r, 3, KMP_ARCH_X86 )
900 ATOMIC_CMPXCHG( float4, mul, kmp_real32, 32, *, 4r, 3, KMP_ARCH_X86 )
901 ATOMIC_CMPXCHG( float8, div, kmp_real64, 64, /, 8r, 7, KMP_ARCH_X86 )
902 ATOMIC_CMPXCHG( float8, mul, kmp_real64, 64, *, 8r, 7, KMP_ARCH_X86 )
913 #define ATOMIC_CRIT_L(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 914 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 915 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \ 916 OP_CRITICAL( = *lhs OP, LCK_ID ) \ 919 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 923 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 924 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 925 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \ 926 OP_CMPXCHG(TYPE,BITS,OP) \ 932 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 933 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 934 OP_GOMP_CRITICAL(= *lhs OP,GOMP_FLAG) \ 935 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 936 OP_CMPXCHG(TYPE,BITS,OP) \ 939 OP_CRITICAL(= *lhs OP,LCK_ID) \ 944 ATOMIC_CMPX_L( fixed1, andl,
char, 8, &&, 1i, 0, KMP_ARCH_X86 )
945 ATOMIC_CMPX_L( fixed1, orl,
char, 8, ||, 1i, 0, KMP_ARCH_X86 )
946 ATOMIC_CMPX_L( fixed2, andl,
short, 16, &&, 2i, 1, KMP_ARCH_X86 )
947 ATOMIC_CMPX_L( fixed2, orl,
short, 16, ||, 2i, 1, KMP_ARCH_X86 )
948 ATOMIC_CMPX_L( fixed4, andl, kmp_int32, 32, &&, 4i, 3, 0 )
949 ATOMIC_CMPX_L( fixed4, orl, kmp_int32, 32, ||, 4i, 3, 0 )
950 ATOMIC_CMPX_L( fixed8, andl, kmp_int64, 64, &&, 8i, 7, KMP_ARCH_X86 )
951 ATOMIC_CMPX_L( fixed8, orl, kmp_int64, 64, ||, 8i, 7, KMP_ARCH_X86 )
964 #define MIN_MAX_CRITSECT(OP,LCK_ID) \ 965 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 967 if ( *lhs OP rhs ) { \ 970 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 973 #ifdef KMP_GOMP_COMPAT 974 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) \ 975 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \ 977 MIN_MAX_CRITSECT( OP, 0 ); \ 981 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) 985 #define MIN_MAX_CMPXCHG(TYPE,BITS,OP) \ 987 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 990 old_value = temp_val; \ 991 while ( old_value OP rhs && \ 992 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 993 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 994 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \ 998 old_value = temp_val; \ 1004 #define MIN_MAX_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1005 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1006 if ( *lhs OP rhs ) { \ 1007 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \ 1008 MIN_MAX_CRITSECT(OP,LCK_ID) \ 1012 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1016 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1017 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1018 if ( *lhs OP rhs ) { \ 1019 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \ 1020 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \ 1027 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1028 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1029 if ( *lhs OP rhs ) { \ 1030 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \ 1031 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1032 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \ 1035 MIN_MAX_CRITSECT(OP,LCK_ID) \ 1041 MIN_MAX_COMPXCHG( fixed1, max,
char, 8, <, 1i, 0, KMP_ARCH_X86 )
1042 MIN_MAX_COMPXCHG( fixed1, min,
char, 8, >, 1i, 0, KMP_ARCH_X86 )
1043 MIN_MAX_COMPXCHG( fixed2, max,
short, 16, <, 2i, 1, KMP_ARCH_X86 )
1044 MIN_MAX_COMPXCHG( fixed2, min,
short, 16, >, 2i, 1, KMP_ARCH_X86 )
1045 MIN_MAX_COMPXCHG( fixed4, max, kmp_int32, 32, <, 4i, 3, 0 )
1046 MIN_MAX_COMPXCHG( fixed4, min, kmp_int32, 32, >, 4i, 3, 0 )
1047 MIN_MAX_COMPXCHG( fixed8, max, kmp_int64, 64, <, 8i, 7, KMP_ARCH_X86 )
1048 MIN_MAX_COMPXCHG( fixed8, min, kmp_int64, 64, >, 8i, 7, KMP_ARCH_X86 )
1049 MIN_MAX_COMPXCHG( float4, max, kmp_real32, 32, <, 4r, 3, KMP_ARCH_X86 )
1050 MIN_MAX_COMPXCHG( float4, min, kmp_real32, 32, >, 4r, 3, KMP_ARCH_X86 )
1051 MIN_MAX_COMPXCHG( float8, max, kmp_real64, 64, <, 8r, 7, KMP_ARCH_X86 )
1052 MIN_MAX_COMPXCHG( float8, min, kmp_real64, 64, >, 8r, 7, KMP_ARCH_X86 )
1054 MIN_MAX_CRITICAL( float16, max, QUAD_LEGACY, <, 16r, 1 )
1055 MIN_MAX_CRITICAL( float16, min, QUAD_LEGACY, >, 16r, 1 )
1056 #if ( KMP_ARCH_X86 ) 1057 MIN_MAX_CRITICAL( float16, max_a16, Quad_a16_t, <, 16r, 1 )
1058 MIN_MAX_CRITICAL( float16, min_a16, Quad_a16_t, >, 16r, 1 )
1064 #define ATOMIC_CRIT_EQV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1065 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1066 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \ 1067 OP_CRITICAL(^=~,LCK_ID) \ 1071 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1074 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1075 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1076 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \ 1077 OP_CMPXCHG(TYPE,BITS,OP) \ 1083 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1084 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1085 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \ 1086 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1087 OP_CMPXCHG(TYPE,BITS,OP) \ 1090 OP_CRITICAL(^=~,LCK_ID) \ 1095 ATOMIC_CMPXCHG( fixed1, neqv, kmp_int8, 8, ^, 1i, 0, KMP_ARCH_X86 )
1096 ATOMIC_CMPXCHG( fixed2, neqv, kmp_int16, 16, ^, 2i, 1, KMP_ARCH_X86 )
1097 ATOMIC_CMPXCHG( fixed4, neqv, kmp_int32, 32, ^, 4i, 3, KMP_ARCH_X86 )
1098 ATOMIC_CMPXCHG( fixed8, neqv, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
1099 ATOMIC_CMPX_EQV( fixed1, eqv, kmp_int8, 8, ^~, 1i, 0, KMP_ARCH_X86 )
1100 ATOMIC_CMPX_EQV( fixed2, eqv, kmp_int16, 16, ^~, 2i, 1, KMP_ARCH_X86 )
1101 ATOMIC_CMPX_EQV( fixed4, eqv, kmp_int32, 32, ^~, 4i, 3, KMP_ARCH_X86 )
1102 ATOMIC_CMPX_EQV( fixed8, eqv, kmp_int64, 64, ^~, 8i, 7, KMP_ARCH_X86 )
1110 #define ATOMIC_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1111 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1112 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1113 OP_CRITICAL(OP##=,LCK_ID) \ 1118 ATOMIC_CRITICAL( float10, add,
long double, +, 10r, 1 )
1119 ATOMIC_CRITICAL( float10, sub,
long double, -, 10r, 1 )
1120 ATOMIC_CRITICAL( float10, mul,
long double, *, 10r, 1 )
1121 ATOMIC_CRITICAL( float10, div,
long double, /, 10r, 1 )
1124 ATOMIC_CRITICAL( float16, add, QUAD_LEGACY, +, 16r, 1 )
1125 ATOMIC_CRITICAL( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1126 ATOMIC_CRITICAL( float16, mul, QUAD_LEGACY, *, 16r, 1 )
1127 ATOMIC_CRITICAL( float16, div, QUAD_LEGACY, /, 16r, 1 )
1128 #if ( KMP_ARCH_X86 ) 1129 ATOMIC_CRITICAL( float16, add_a16, Quad_a16_t, +, 16r, 1 )
1130 ATOMIC_CRITICAL( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1131 ATOMIC_CRITICAL( float16, mul_a16, Quad_a16_t, *, 16r, 1 )
1132 ATOMIC_CRITICAL( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1139 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, add, kmp_cmplx32, 64, +, 8c, 7, 1 )
1140 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7, 1 )
1141 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7, 1 )
1142 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, div, kmp_cmplx32, 64, /, 8c, 7, 1 )
1145 ATOMIC_CRITICAL( cmplx4, add, kmp_cmplx32, +, 8c, 1 )
1146 ATOMIC_CRITICAL( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1147 ATOMIC_CRITICAL( cmplx4, mul, kmp_cmplx32, *, 8c, 1 )
1148 ATOMIC_CRITICAL( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1149 #endif // USE_CMPXCHG_FIX 1151 ATOMIC_CRITICAL( cmplx8, add, kmp_cmplx64, +, 16c, 1 )
1152 ATOMIC_CRITICAL( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1153 ATOMIC_CRITICAL( cmplx8, mul, kmp_cmplx64, *, 16c, 1 )
1154 ATOMIC_CRITICAL( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1155 ATOMIC_CRITICAL( cmplx10, add, kmp_cmplx80, +, 20c, 1 )
1156 ATOMIC_CRITICAL( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1157 ATOMIC_CRITICAL( cmplx10, mul, kmp_cmplx80, *, 20c, 1 )
1158 ATOMIC_CRITICAL( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1160 ATOMIC_CRITICAL( cmplx16, add, CPLX128_LEG, +, 32c, 1 )
1161 ATOMIC_CRITICAL( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1162 ATOMIC_CRITICAL( cmplx16, mul, CPLX128_LEG, *, 32c, 1 )
1163 ATOMIC_CRITICAL( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1164 #if ( KMP_ARCH_X86 ) 1165 ATOMIC_CRITICAL( cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c, 1 )
1166 ATOMIC_CRITICAL( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1167 ATOMIC_CRITICAL( cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c, 1 )
1168 ATOMIC_CRITICAL( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1176 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1184 #define OP_CRITICAL_REV(OP,LCK_ID) \ 1185 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1187 (*lhs) = (rhs) OP (*lhs); \ 1189 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 1191 #ifdef KMP_GOMP_COMPAT 1192 #define OP_GOMP_CRITICAL_REV(OP,FLAG) \ 1193 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1195 OP_CRITICAL_REV( OP, 0 ); \ 1199 #define OP_GOMP_CRITICAL_REV(OP,FLAG) 1207 #define ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE, RET_TYPE) \ 1208 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \ 1210 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1211 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid )); 1220 #define OP_CMPXCHG_REV(TYPE,BITS,OP) \ 1222 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1223 TYPE old_value, new_value; \ 1225 old_value = temp_val; \ 1226 new_value = rhs OP old_value; \ 1227 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 1228 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 1229 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 1234 old_value = temp_val; \ 1235 new_value = rhs OP old_value; \ 1240 #define ATOMIC_CMPXCHG_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,GOMP_FLAG) \ 1241 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \ 1242 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \ 1243 OP_CMPXCHG_REV(TYPE,BITS,OP) \ 1260 ATOMIC_CMPXCHG_REV( fixed1, div, kmp_int8, 8, /, 1i, KMP_ARCH_X86 )
1261 ATOMIC_CMPXCHG_REV( fixed1u, div, kmp_uint8, 8, /, 1i, KMP_ARCH_X86 )
1262 ATOMIC_CMPXCHG_REV( fixed1, shl, kmp_int8, 8, <<, 1i, KMP_ARCH_X86 )
1263 ATOMIC_CMPXCHG_REV( fixed1, shr, kmp_int8, 8, >>, 1i, KMP_ARCH_X86 )
1264 ATOMIC_CMPXCHG_REV( fixed1u, shr, kmp_uint8, 8, >>, 1i, KMP_ARCH_X86 )
1265 ATOMIC_CMPXCHG_REV( fixed1, sub, kmp_int8, 8, -, 1i, KMP_ARCH_X86 )
1267 ATOMIC_CMPXCHG_REV( fixed2, div, kmp_int16, 16, /, 2i, KMP_ARCH_X86 )
1268 ATOMIC_CMPXCHG_REV( fixed2u, div, kmp_uint16, 16, /, 2i, KMP_ARCH_X86 )
1269 ATOMIC_CMPXCHG_REV( fixed2, shl, kmp_int16, 16, <<, 2i, KMP_ARCH_X86 )
1270 ATOMIC_CMPXCHG_REV( fixed2, shr, kmp_int16, 16, >>, 2i, KMP_ARCH_X86 )
1271 ATOMIC_CMPXCHG_REV( fixed2u, shr, kmp_uint16, 16, >>, 2i, KMP_ARCH_X86 )
1272 ATOMIC_CMPXCHG_REV( fixed2, sub, kmp_int16, 16, -, 2i, KMP_ARCH_X86 )
1274 ATOMIC_CMPXCHG_REV( fixed4, div, kmp_int32, 32, /, 4i, KMP_ARCH_X86 )
1275 ATOMIC_CMPXCHG_REV( fixed4u, div, kmp_uint32, 32, /, 4i, KMP_ARCH_X86 )
1276 ATOMIC_CMPXCHG_REV( fixed4, shl, kmp_int32, 32, <<, 4i, KMP_ARCH_X86 )
1277 ATOMIC_CMPXCHG_REV( fixed4, shr, kmp_int32, 32, >>, 4i, KMP_ARCH_X86 )
1278 ATOMIC_CMPXCHG_REV( fixed4u, shr, kmp_uint32, 32, >>, 4i, KMP_ARCH_X86 )
1279 ATOMIC_CMPXCHG_REV( fixed4, sub, kmp_int32, 32, -, 4i, KMP_ARCH_X86 )
1281 ATOMIC_CMPXCHG_REV( fixed8, div, kmp_int64, 64, /, 8i, KMP_ARCH_X86 )
1282 ATOMIC_CMPXCHG_REV( fixed8u, div, kmp_uint64, 64, /, 8i, KMP_ARCH_X86 )
1283 ATOMIC_CMPXCHG_REV( fixed8, shl, kmp_int64, 64, <<, 8i, KMP_ARCH_X86 )
1284 ATOMIC_CMPXCHG_REV( fixed8, shr, kmp_int64, 64, >>, 8i, KMP_ARCH_X86 )
1285 ATOMIC_CMPXCHG_REV( fixed8u, shr, kmp_uint64, 64, >>, 8i, KMP_ARCH_X86 )
1286 ATOMIC_CMPXCHG_REV( fixed8, sub, kmp_int64, 64, -, 8i, KMP_ARCH_X86 )
1288 ATOMIC_CMPXCHG_REV( float4, div, kmp_real32, 32, /, 4r, KMP_ARCH_X86 )
1289 ATOMIC_CMPXCHG_REV( float4, sub, kmp_real32, 32, -, 4r, KMP_ARCH_X86 )
1291 ATOMIC_CMPXCHG_REV( float8, div, kmp_real64, 64, /, 8r, KMP_ARCH_X86 )
1292 ATOMIC_CMPXCHG_REV( float8, sub, kmp_real64, 64, -, 8r, KMP_ARCH_X86 )
1300 #define ATOMIC_CRITICAL_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1301 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \ 1302 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \ 1303 OP_CRITICAL_REV(OP,LCK_ID) \ 1308 ATOMIC_CRITICAL_REV( float10, sub,
long double, -, 10r, 1 )
1309 ATOMIC_CRITICAL_REV( float10, div,
long double, /, 10r, 1 )
1312 ATOMIC_CRITICAL_REV( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1313 ATOMIC_CRITICAL_REV( float16, div, QUAD_LEGACY, /, 16r, 1 )
1314 #if ( KMP_ARCH_X86 ) 1315 ATOMIC_CRITICAL_REV( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1316 ATOMIC_CRITICAL_REV( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1321 ATOMIC_CRITICAL_REV( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1322 ATOMIC_CRITICAL_REV( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1323 ATOMIC_CRITICAL_REV( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1324 ATOMIC_CRITICAL_REV( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1325 ATOMIC_CRITICAL_REV( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1326 ATOMIC_CRITICAL_REV( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1328 ATOMIC_CRITICAL_REV( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1329 ATOMIC_CRITICAL_REV( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1330 #if ( KMP_ARCH_X86 ) 1331 ATOMIC_CRITICAL_REV( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1332 ATOMIC_CRITICAL_REV( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1337 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64 1340 #endif //OMP_40_ENABLED 1354 #define ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1355 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs ) \ 1357 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1358 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid )); 1361 #define ATOMIC_CRITICAL_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \ 1362 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1363 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1364 OP_CRITICAL(OP##=,LCK_ID) \ 1368 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1371 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1372 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1373 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1374 OP_CMPXCHG(TYPE,BITS,OP) \ 1380 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1381 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1382 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1383 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1384 OP_CMPXCHG(TYPE,BITS,OP) \ 1387 OP_CRITICAL(OP##=,LCK_ID) \ 1393 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1395 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1396 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1397 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \ 1398 OP_CMPXCHG_REV(TYPE,BITS,OP) \ 1400 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \ 1401 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1402 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \ 1403 OP_CRITICAL_REV(OP,LCK_ID) \ 1408 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1409 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1410 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1411 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1412 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3, 0 )
1413 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3, 0 )
1414 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1415 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1416 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1417 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1418 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1419 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1423 ATOMIC_CMPXCHG_MIX( fixed1,
char, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1424 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1425 ATOMIC_CMPXCHG_MIX( fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1426 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1427 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1428 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1429 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1430 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1432 ATOMIC_CMPXCHG_MIX( fixed2,
short, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1433 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1434 ATOMIC_CMPXCHG_MIX( fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1435 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1436 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1437 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1438 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1439 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1441 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3, 0 )
1442 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3, 0 )
1443 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3, 0 )
1444 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3, 0 )
1445 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3, 0 )
1446 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3, 0 )
1447 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1448 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1450 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1451 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1452 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1453 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1454 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1455 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1456 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1457 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1459 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1460 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1461 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1462 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1464 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1465 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1466 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1467 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1469 ATOMIC_CRITICAL_FP( float10,
long double, add, +, fp, _Quad, 10r, 1 )
1470 ATOMIC_CRITICAL_FP( float10,
long double, sub, -, fp, _Quad, 10r, 1 )
1471 ATOMIC_CRITICAL_FP( float10,
long double, mul, *, fp, _Quad, 10r, 1 )
1472 ATOMIC_CRITICAL_FP( float10,
long double, div, /, fp, _Quad, 10r, 1 )
1474 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1476 ATOMIC_CMPXCHG_REV_MIX( fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1477 ATOMIC_CMPXCHG_REV_MIX( fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1478 ATOMIC_CMPXCHG_REV_MIX( fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1479 ATOMIC_CMPXCHG_REV_MIX( fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1481 ATOMIC_CMPXCHG_REV_MIX( fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1482 ATOMIC_CMPXCHG_REV_MIX( fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1483 ATOMIC_CMPXCHG_REV_MIX( fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1484 ATOMIC_CMPXCHG_REV_MIX( fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1486 ATOMIC_CMPXCHG_REV_MIX( fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3, 0 )
1487 ATOMIC_CMPXCHG_REV_MIX( fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3, 0 )
1488 ATOMIC_CMPXCHG_REV_MIX( fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3, 0 )
1489 ATOMIC_CMPXCHG_REV_MIX( fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3, 0 )
1491 ATOMIC_CMPXCHG_REV_MIX( fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1492 ATOMIC_CMPXCHG_REV_MIX( fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1493 ATOMIC_CMPXCHG_REV_MIX( fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1494 ATOMIC_CMPXCHG_REV_MIX( fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1496 ATOMIC_CMPXCHG_REV_MIX( float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1497 ATOMIC_CMPXCHG_REV_MIX( float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1499 ATOMIC_CMPXCHG_REV_MIX( float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1500 ATOMIC_CMPXCHG_REV_MIX( float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1502 ATOMIC_CRITICAL_REV_FP( float10,
long double, sub_rev, -, fp, _Quad, 10r, 1 )
1503 ATOMIC_CRITICAL_REV_FP( float10,
long double, div_rev, /, fp, _Quad, 10r, 1 )
1508 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1513 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1514 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1515 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1516 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \ 1520 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1521 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1522 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1523 OP_CMPXCHG(TYPE,BITS,OP) \ 1525 #endif // USE_CMPXCHG_FIX 1529 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1530 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1531 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1532 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1533 OP_CMPXCHG(TYPE,BITS,OP) \ 1536 OP_CRITICAL(OP##=,LCK_ID) \ 1541 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1542 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1543 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1544 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1547 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1559 #define ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE, RET_TYPE) \ 1560 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * loc ) \ 1562 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1563 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 1575 #define OP_CMPXCHG_READ(TYPE,BITS,OP) \ 1577 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1580 kmp_int##BITS i_val; \ 1582 union f_i_union old_value; \ 1584 old_value.f_val = temp_val; \ 1585 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( (kmp_int##BITS *) loc, \ 1586 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val, \ 1587 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val ); \ 1588 new_value = old_value.f_val; \ 1598 #define OP_CRITICAL_READ(OP,LCK_ID) \ 1599 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1601 new_value = (*loc); \ 1603 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 1606 #ifdef KMP_GOMP_COMPAT 1607 #define OP_GOMP_CRITICAL_READ(OP,FLAG) \ 1608 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1610 OP_CRITICAL_READ( OP, 0 ); \ 1614 #define OP_GOMP_CRITICAL_READ(OP,FLAG) 1618 #define ATOMIC_FIXED_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1619 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \ 1621 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \ 1622 new_value = KMP_TEST_THEN_ADD##BITS( loc, OP 0 ); \ 1626 #define ATOMIC_CMPXCHG_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1627 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \ 1629 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \ 1630 OP_CMPXCHG_READ(TYPE,BITS,OP) \ 1637 #define ATOMIC_CRITICAL_READ(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1638 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \ 1640 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \ 1641 OP_CRITICAL_READ(OP,LCK_ID) \ 1649 #if ( KMP_OS_WINDOWS ) 1651 #define OP_CRITICAL_READ_WRK(OP,LCK_ID) \ 1652 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1656 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 1658 #ifdef KMP_GOMP_COMPAT 1659 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) \ 1660 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1662 OP_CRITICAL_READ_WRK( OP, 0 ); \ 1665 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) 1668 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \ 1669 void __kmpc_atomic_##TYPE_ID##_##OP_ID( TYPE * out, ident_t *id_ref, int gtid, TYPE * loc ) \ 1671 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1672 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 1675 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1676 ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \ 1677 OP_GOMP_CRITICAL_READ_WRK(OP##=,GOMP_FLAG) \ 1678 OP_CRITICAL_READ_WRK(OP,LCK_ID) \ 1681 #endif // KMP_OS_WINDOWS 1685 ATOMIC_FIXED_READ( fixed4, rd, kmp_int32, 32, +, 0 )
1686 ATOMIC_FIXED_READ( fixed8, rd, kmp_int64, 64, +, KMP_ARCH_X86 )
1687 ATOMIC_CMPXCHG_READ( float4, rd, kmp_real32, 32, +, KMP_ARCH_X86 )
1688 ATOMIC_CMPXCHG_READ( float8, rd, kmp_real64, 64, +, KMP_ARCH_X86 )
1691 ATOMIC_CMPXCHG_READ( fixed1, rd, kmp_int8, 8, +, KMP_ARCH_X86 )
1692 ATOMIC_CMPXCHG_READ( fixed2, rd, kmp_int16, 16, +, KMP_ARCH_X86 )
1694 ATOMIC_CRITICAL_READ( float10, rd,
long double, +, 10r, 1 )
1696 ATOMIC_CRITICAL_READ( float16, rd, QUAD_LEGACY, +, 16r, 1 )
1697 #endif // KMP_HAVE_QUAD 1700 #if ( KMP_OS_WINDOWS ) 1701 ATOMIC_CRITICAL_READ_WRK( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1703 ATOMIC_CRITICAL_READ( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1705 ATOMIC_CRITICAL_READ( cmplx8, rd, kmp_cmplx64, +, 16c, 1 )
1706 ATOMIC_CRITICAL_READ( cmplx10, rd, kmp_cmplx80, +, 20c, 1 )
1708 ATOMIC_CRITICAL_READ( cmplx16, rd, CPLX128_LEG, +, 32c, 1 )
1709 #if ( KMP_ARCH_X86 ) 1710 ATOMIC_CRITICAL_READ( float16, a16_rd, Quad_a16_t, +, 16r, 1 )
1711 ATOMIC_CRITICAL_READ( cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c, 1 )
1720 #define ATOMIC_XCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1721 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1722 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1723 KMP_XCHG_FIXED##BITS( lhs, rhs ); \ 1726 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1727 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1728 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1729 KMP_XCHG_REAL##BITS( lhs, rhs ); \ 1740 #define OP_CMPXCHG_WR(TYPE,BITS,OP) \ 1742 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1743 TYPE old_value, new_value; \ 1745 old_value = temp_val; \ 1747 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 1748 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 1749 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 1754 old_value = temp_val; \ 1760 #define ATOMIC_CMPXCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1761 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1762 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1763 OP_CMPXCHG_WR(TYPE,BITS,OP) \ 1771 #define ATOMIC_CRITICAL_WR(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1772 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1773 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1774 OP_CRITICAL(OP,LCK_ID) \ 1778 ATOMIC_XCHG_WR( fixed1, wr, kmp_int8, 8, =, KMP_ARCH_X86 )
1779 ATOMIC_XCHG_WR( fixed2, wr, kmp_int16, 16, =, KMP_ARCH_X86 )
1780 ATOMIC_XCHG_WR( fixed4, wr, kmp_int32, 32, =, KMP_ARCH_X86 )
1781 #if ( KMP_ARCH_X86 ) 1782 ATOMIC_CMPXCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1784 ATOMIC_XCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1787 ATOMIC_XCHG_FLOAT_WR( float4, wr, kmp_real32, 32, =, KMP_ARCH_X86 )
1788 #if ( KMP_ARCH_X86 ) 1789 ATOMIC_CMPXCHG_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1791 ATOMIC_XCHG_FLOAT_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1794 ATOMIC_CRITICAL_WR( float10, wr,
long double, =, 10r, 1 )
1796 ATOMIC_CRITICAL_WR( float16, wr, QUAD_LEGACY, =, 16r, 1 )
1798 ATOMIC_CRITICAL_WR( cmplx4, wr, kmp_cmplx32, =, 8c, 1 )
1799 ATOMIC_CRITICAL_WR( cmplx8, wr, kmp_cmplx64, =, 16c, 1 )
1800 ATOMIC_CRITICAL_WR( cmplx10, wr, kmp_cmplx80, =, 20c, 1 )
1802 ATOMIC_CRITICAL_WR( cmplx16, wr, CPLX128_LEG, =, 32c, 1 )
1803 #if ( KMP_ARCH_X86 ) 1804 ATOMIC_CRITICAL_WR( float16, a16_wr, Quad_a16_t, =, 16r, 1 )
1805 ATOMIC_CRITICAL_WR( cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c, 1 )
1818 #define ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,RET_TYPE) \ 1819 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag ) \ 1821 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1822 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 1830 #define OP_CRITICAL_CPT(OP,LCK_ID) \ 1831 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1835 new_value = (*lhs); \ 1837 new_value = (*lhs); \ 1841 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1845 #ifdef KMP_GOMP_COMPAT 1846 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) \ 1847 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1849 OP_CRITICAL_CPT( OP##=, 0 ); \ 1852 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) 1862 #define OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 1864 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1865 TYPE old_value, new_value; \ 1867 old_value = temp_val; \ 1868 new_value = old_value OP rhs; \ 1869 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 1870 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 1871 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 1876 old_value = temp_val; \ 1877 new_value = old_value OP rhs; \ 1886 #define ATOMIC_CMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1887 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 1889 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 1890 OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 1894 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1895 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 1896 TYPE old_value, new_value; \ 1897 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 1899 old_value = KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \ 1901 return old_value OP rhs; \ 1907 ATOMIC_FIXED_ADD_CPT( fixed4, add_cpt, kmp_int32, 32, +, 0 )
1908 ATOMIC_FIXED_ADD_CPT( fixed4, sub_cpt, kmp_int32, 32, -, 0 )
1909 ATOMIC_FIXED_ADD_CPT( fixed8, add_cpt, kmp_int64, 64, +, KMP_ARCH_X86 )
1910 ATOMIC_FIXED_ADD_CPT( fixed8, sub_cpt, kmp_int64, 64, -, KMP_ARCH_X86 )
1912 ATOMIC_CMPXCHG_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1913 ATOMIC_CMPXCHG_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1914 ATOMIC_CMPXCHG_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1915 ATOMIC_CMPXCHG_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1929 ATOMIC_CMPXCHG_CPT( fixed1, add_cpt, kmp_int8, 8, +, KMP_ARCH_X86 )
1930 ATOMIC_CMPXCHG_CPT( fixed1, andb_cpt, kmp_int8, 8, &, 0 )
1931 ATOMIC_CMPXCHG_CPT( fixed1, div_cpt, kmp_int8, 8, /, KMP_ARCH_X86 )
1932 ATOMIC_CMPXCHG_CPT( fixed1u, div_cpt, kmp_uint8, 8, /, KMP_ARCH_X86 )
1933 ATOMIC_CMPXCHG_CPT( fixed1, mul_cpt, kmp_int8, 8, *, KMP_ARCH_X86 )
1934 ATOMIC_CMPXCHG_CPT( fixed1, orb_cpt, kmp_int8, 8, |, 0 )
1935 ATOMIC_CMPXCHG_CPT( fixed1, shl_cpt, kmp_int8, 8, <<, KMP_ARCH_X86 )
1936 ATOMIC_CMPXCHG_CPT( fixed1, shr_cpt, kmp_int8, 8, >>, KMP_ARCH_X86 )
1937 ATOMIC_CMPXCHG_CPT( fixed1u, shr_cpt, kmp_uint8, 8, >>, KMP_ARCH_X86 )
1938 ATOMIC_CMPXCHG_CPT( fixed1, sub_cpt, kmp_int8, 8, -, KMP_ARCH_X86 )
1939 ATOMIC_CMPXCHG_CPT( fixed1, xor_cpt, kmp_int8, 8, ^, 0 )
1940 ATOMIC_CMPXCHG_CPT( fixed2, add_cpt, kmp_int16, 16, +, KMP_ARCH_X86 )
1941 ATOMIC_CMPXCHG_CPT( fixed2, andb_cpt, kmp_int16, 16, &, 0 )
1942 ATOMIC_CMPXCHG_CPT( fixed2, div_cpt, kmp_int16, 16, /, KMP_ARCH_X86 )
1943 ATOMIC_CMPXCHG_CPT( fixed2u, div_cpt, kmp_uint16, 16, /, KMP_ARCH_X86 )
1944 ATOMIC_CMPXCHG_CPT( fixed2, mul_cpt, kmp_int16, 16, *, KMP_ARCH_X86 )
1945 ATOMIC_CMPXCHG_CPT( fixed2, orb_cpt, kmp_int16, 16, |, 0 )
1946 ATOMIC_CMPXCHG_CPT( fixed2, shl_cpt, kmp_int16, 16, <<, KMP_ARCH_X86 )
1947 ATOMIC_CMPXCHG_CPT( fixed2, shr_cpt, kmp_int16, 16, >>, KMP_ARCH_X86 )
1948 ATOMIC_CMPXCHG_CPT( fixed2u, shr_cpt, kmp_uint16, 16, >>, KMP_ARCH_X86 )
1949 ATOMIC_CMPXCHG_CPT( fixed2, sub_cpt, kmp_int16, 16, -, KMP_ARCH_X86 )
1950 ATOMIC_CMPXCHG_CPT( fixed2, xor_cpt, kmp_int16, 16, ^, 0 )
1951 ATOMIC_CMPXCHG_CPT( fixed4, andb_cpt, kmp_int32, 32, &, 0 )
1952 ATOMIC_CMPXCHG_CPT( fixed4, div_cpt, kmp_int32, 32, /, KMP_ARCH_X86 )
1953 ATOMIC_CMPXCHG_CPT( fixed4u, div_cpt, kmp_uint32, 32, /, KMP_ARCH_X86 )
1954 ATOMIC_CMPXCHG_CPT( fixed4, mul_cpt, kmp_int32, 32, *, KMP_ARCH_X86 )
1955 ATOMIC_CMPXCHG_CPT( fixed4, orb_cpt, kmp_int32, 32, |, 0 )
1956 ATOMIC_CMPXCHG_CPT( fixed4, shl_cpt, kmp_int32, 32, <<, KMP_ARCH_X86 )
1957 ATOMIC_CMPXCHG_CPT( fixed4, shr_cpt, kmp_int32, 32, >>, KMP_ARCH_X86 )
1958 ATOMIC_CMPXCHG_CPT( fixed4u, shr_cpt, kmp_uint32, 32, >>, KMP_ARCH_X86 )
1959 ATOMIC_CMPXCHG_CPT( fixed4, xor_cpt, kmp_int32, 32, ^, 0 )
1960 ATOMIC_CMPXCHG_CPT( fixed8, andb_cpt, kmp_int64, 64, &, KMP_ARCH_X86 )
1961 ATOMIC_CMPXCHG_CPT( fixed8, div_cpt, kmp_int64, 64, /, KMP_ARCH_X86 )
1962 ATOMIC_CMPXCHG_CPT( fixed8u, div_cpt, kmp_uint64, 64, /, KMP_ARCH_X86 )
1963 ATOMIC_CMPXCHG_CPT( fixed8, mul_cpt, kmp_int64, 64, *, KMP_ARCH_X86 )
1964 ATOMIC_CMPXCHG_CPT( fixed8, orb_cpt, kmp_int64, 64, |, KMP_ARCH_X86 )
1965 ATOMIC_CMPXCHG_CPT( fixed8, shl_cpt, kmp_int64, 64, <<, KMP_ARCH_X86 )
1966 ATOMIC_CMPXCHG_CPT( fixed8, shr_cpt, kmp_int64, 64, >>, KMP_ARCH_X86 )
1967 ATOMIC_CMPXCHG_CPT( fixed8u, shr_cpt, kmp_uint64, 64, >>, KMP_ARCH_X86 )
1968 ATOMIC_CMPXCHG_CPT( fixed8, xor_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
1969 ATOMIC_CMPXCHG_CPT( float4, div_cpt, kmp_real32, 32, /, KMP_ARCH_X86 )
1970 ATOMIC_CMPXCHG_CPT( float4, mul_cpt, kmp_real32, 32, *, KMP_ARCH_X86 )
1971 ATOMIC_CMPXCHG_CPT( float8, div_cpt, kmp_real64, 64, /, KMP_ARCH_X86 )
1972 ATOMIC_CMPXCHG_CPT( float8, mul_cpt, kmp_real64, 64, *, KMP_ARCH_X86 )
1984 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \ 1985 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs, int flag ) \ 1987 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1988 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid )); 1991 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1992 ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \ 1994 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 1995 OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 1999 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \ 2000 ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \ 2002 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 2003 OP_CRITICAL_CPT(OP##=,LCK_ID) \ 2006 ATOMIC_CMPXCHG_CPT_MIX( fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2007 ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2008 ATOMIC_CMPXCHG_CPT_MIX( fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2009 ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2010 ATOMIC_CMPXCHG_CPT_MIX( fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2011 ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2012 ATOMIC_CMPXCHG_CPT_MIX( fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2013 ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2015 ATOMIC_CMPXCHG_CPT_MIX( fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2016 ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2017 ATOMIC_CMPXCHG_CPT_MIX( fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2018 ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2019 ATOMIC_CMPXCHG_CPT_MIX( fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2020 ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2021 ATOMIC_CMPXCHG_CPT_MIX( fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2022 ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2024 ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3, 0 )
2025 ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3, 0 )
2026 ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3, 0 )
2027 ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3, 0 )
2028 ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3, 0 )
2029 ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3, 0 )
2030 ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3, 0 )
2031 ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3, 0 )
2033 ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2034 ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2035 ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2036 ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2037 ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2038 ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2039 ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2040 ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2042 ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
2043 ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
2044 ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
2045 ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
2047 ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
2048 ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
2049 ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
2050 ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
2052 ATOMIC_CRITICAL_CPT_MIX( float10,
long double, add_cpt, +, fp, _Quad, 10r, 1 )
2053 ATOMIC_CRITICAL_CPT_MIX( float10,
long double, sub_cpt, -, fp, _Quad, 10r, 1 )
2054 ATOMIC_CRITICAL_CPT_MIX( float10,
long double, mul_cpt, *, fp, _Quad, 10r, 1 )
2055 ATOMIC_CRITICAL_CPT_MIX( float10,
long double, div_cpt, /, fp, _Quad, 10r, 1 )
2057 #endif //KMP_HAVE_QUAD 2071 #define OP_CRITICAL_L_CPT(OP,LCK_ID) \ 2072 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2077 new_value = (*lhs); \ 2079 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 2082 #ifdef KMP_GOMP_COMPAT 2083 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) \ 2084 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2086 OP_CRITICAL_L_CPT( OP, 0 ); \ 2090 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) 2095 #define ATOMIC_CMPX_L_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 2096 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2098 OP_GOMP_CRITICAL_L_CPT( = *lhs OP, GOMP_FLAG ) \ 2099 OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 2102 ATOMIC_CMPX_L_CPT( fixed1, andl_cpt,
char, 8, &&, KMP_ARCH_X86 )
2103 ATOMIC_CMPX_L_CPT( fixed1, orl_cpt,
char, 8, ||, KMP_ARCH_X86 )
2104 ATOMIC_CMPX_L_CPT( fixed2, andl_cpt,
short, 16, &&, KMP_ARCH_X86 )
2105 ATOMIC_CMPX_L_CPT( fixed2, orl_cpt,
short, 16, ||, KMP_ARCH_X86 )
2106 ATOMIC_CMPX_L_CPT( fixed4, andl_cpt, kmp_int32, 32, &&, 0 )
2107 ATOMIC_CMPX_L_CPT( fixed4, orl_cpt, kmp_int32, 32, ||, 0 )
2108 ATOMIC_CMPX_L_CPT( fixed8, andl_cpt, kmp_int64, 64, &&, KMP_ARCH_X86 )
2109 ATOMIC_CMPX_L_CPT( fixed8, orl_cpt, kmp_int64, 64, ||, KMP_ARCH_X86 )
2122 #define MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \ 2123 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2125 if ( *lhs OP rhs ) { \ 2131 new_value = old_value; \ 2133 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2137 #ifdef KMP_GOMP_COMPAT 2138 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) \ 2139 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \ 2141 MIN_MAX_CRITSECT_CPT( OP, 0 ); \ 2144 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) 2148 #define MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \ 2150 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2153 old_value = temp_val; \ 2154 while ( old_value OP rhs && \ 2155 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 2156 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 2157 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \ 2161 old_value = temp_val; \ 2171 #define MIN_MAX_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2172 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2173 TYPE new_value, old_value; \ 2174 if ( *lhs OP rhs ) { \ 2175 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \ 2176 MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \ 2181 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 2182 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2183 TYPE new_value, old_value; \ 2184 if ( *lhs OP rhs ) { \ 2185 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \ 2186 MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \ 2192 MIN_MAX_COMPXCHG_CPT( fixed1, max_cpt,
char, 8, <, KMP_ARCH_X86 )
2193 MIN_MAX_COMPXCHG_CPT( fixed1, min_cpt,
char, 8, >, KMP_ARCH_X86 )
2194 MIN_MAX_COMPXCHG_CPT( fixed2, max_cpt,
short, 16, <, KMP_ARCH_X86 )
2195 MIN_MAX_COMPXCHG_CPT( fixed2, min_cpt,
short, 16, >, KMP_ARCH_X86 )
2196 MIN_MAX_COMPXCHG_CPT( fixed4, max_cpt, kmp_int32, 32, <, 0 )
2197 MIN_MAX_COMPXCHG_CPT( fixed4, min_cpt, kmp_int32, 32, >, 0 )
2198 MIN_MAX_COMPXCHG_CPT( fixed8, max_cpt, kmp_int64, 64, <, KMP_ARCH_X86 )
2199 MIN_MAX_COMPXCHG_CPT( fixed8, min_cpt, kmp_int64, 64, >, KMP_ARCH_X86 )
2200 MIN_MAX_COMPXCHG_CPT( float4, max_cpt, kmp_real32, 32, <, KMP_ARCH_X86 )
2201 MIN_MAX_COMPXCHG_CPT( float4, min_cpt, kmp_real32, 32, >, KMP_ARCH_X86 )
2202 MIN_MAX_COMPXCHG_CPT( float8, max_cpt, kmp_real64, 64, <, KMP_ARCH_X86 )
2203 MIN_MAX_COMPXCHG_CPT( float8, min_cpt, kmp_real64, 64, >, KMP_ARCH_X86 )
2205 MIN_MAX_CRITICAL_CPT( float16, max_cpt, QUAD_LEGACY, <, 16r, 1 )
2206 MIN_MAX_CRITICAL_CPT( float16, min_cpt, QUAD_LEGACY, >, 16r, 1 )
2207 #if ( KMP_ARCH_X86 ) 2208 MIN_MAX_CRITICAL_CPT( float16, max_a16_cpt, Quad_a16_t, <, 16r, 1 )
2209 MIN_MAX_CRITICAL_CPT( float16, min_a16_cpt, Quad_a16_t, >, 16r, 1 )
2214 #ifdef KMP_GOMP_COMPAT 2215 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) \ 2216 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2218 OP_CRITICAL_CPT( OP, 0 ); \ 2221 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) 2224 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 2225 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2227 OP_GOMP_CRITICAL_EQV_CPT(^=~,GOMP_FLAG) \ 2228 OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 2233 ATOMIC_CMPXCHG_CPT( fixed1, neqv_cpt, kmp_int8, 8, ^, KMP_ARCH_X86 )
2234 ATOMIC_CMPXCHG_CPT( fixed2, neqv_cpt, kmp_int16, 16, ^, KMP_ARCH_X86 )
2235 ATOMIC_CMPXCHG_CPT( fixed4, neqv_cpt, kmp_int32, 32, ^, KMP_ARCH_X86 )
2236 ATOMIC_CMPXCHG_CPT( fixed8, neqv_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
2237 ATOMIC_CMPX_EQV_CPT( fixed1, eqv_cpt, kmp_int8, 8, ^~, KMP_ARCH_X86 )
2238 ATOMIC_CMPX_EQV_CPT( fixed2, eqv_cpt, kmp_int16, 16, ^~, KMP_ARCH_X86 )
2239 ATOMIC_CMPX_EQV_CPT( fixed4, eqv_cpt, kmp_int32, 32, ^~, KMP_ARCH_X86 )
2240 ATOMIC_CMPX_EQV_CPT( fixed8, eqv_cpt, kmp_int64, 64, ^~, KMP_ARCH_X86 )
2247 #define ATOMIC_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2248 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2250 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 2251 OP_CRITICAL_CPT(OP##=,LCK_ID) \ 2258 #define OP_CRITICAL_CPT_WRK(OP,LCK_ID) \ 2259 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2269 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2273 #ifdef KMP_GOMP_COMPAT 2274 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) \ 2275 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2277 OP_CRITICAL_CPT_WRK( OP##=, 0 ); \ 2280 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) 2284 #define ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \ 2285 void __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out, int flag ) \ 2287 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 2288 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 2291 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2292 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \ 2293 OP_GOMP_CRITICAL_CPT_WRK(OP,GOMP_FLAG) \ 2294 OP_CRITICAL_CPT_WRK(OP##=,LCK_ID) \ 2300 ATOMIC_CRITICAL_CPT( float10, add_cpt,
long double, +, 10r, 1 )
2301 ATOMIC_CRITICAL_CPT( float10, sub_cpt,
long double, -, 10r, 1 )
2302 ATOMIC_CRITICAL_CPT( float10, mul_cpt,
long double, *, 10r, 1 )
2303 ATOMIC_CRITICAL_CPT( float10, div_cpt,
long double, /, 10r, 1 )
2306 ATOMIC_CRITICAL_CPT( float16, add_cpt, QUAD_LEGACY, +, 16r, 1 )
2307 ATOMIC_CRITICAL_CPT( float16, sub_cpt, QUAD_LEGACY, -, 16r, 1 )
2308 ATOMIC_CRITICAL_CPT( float16, mul_cpt, QUAD_LEGACY, *, 16r, 1 )
2309 ATOMIC_CRITICAL_CPT( float16, div_cpt, QUAD_LEGACY, /, 16r, 1 )
2310 #if ( KMP_ARCH_X86 ) 2311 ATOMIC_CRITICAL_CPT( float16, add_a16_cpt, Quad_a16_t, +, 16r, 1 )
2312 ATOMIC_CRITICAL_CPT( float16, sub_a16_cpt, Quad_a16_t, -, 16r, 1 )
2313 ATOMIC_CRITICAL_CPT( float16, mul_a16_cpt, Quad_a16_t, *, 16r, 1 )
2314 ATOMIC_CRITICAL_CPT( float16, div_a16_cpt, Quad_a16_t, /, 16r, 1 )
2321 ATOMIC_CRITICAL_CPT_WRK( cmplx4, add_cpt, kmp_cmplx32, +, 8c, 1 )
2322 ATOMIC_CRITICAL_CPT_WRK( cmplx4, sub_cpt, kmp_cmplx32, -, 8c, 1 )
2323 ATOMIC_CRITICAL_CPT_WRK( cmplx4, mul_cpt, kmp_cmplx32, *, 8c, 1 )
2324 ATOMIC_CRITICAL_CPT_WRK( cmplx4, div_cpt, kmp_cmplx32, /, 8c, 1 )
2326 ATOMIC_CRITICAL_CPT( cmplx8, add_cpt, kmp_cmplx64, +, 16c, 1 )
2327 ATOMIC_CRITICAL_CPT( cmplx8, sub_cpt, kmp_cmplx64, -, 16c, 1 )
2328 ATOMIC_CRITICAL_CPT( cmplx8, mul_cpt, kmp_cmplx64, *, 16c, 1 )
2329 ATOMIC_CRITICAL_CPT( cmplx8, div_cpt, kmp_cmplx64, /, 16c, 1 )
2330 ATOMIC_CRITICAL_CPT( cmplx10, add_cpt, kmp_cmplx80, +, 20c, 1 )
2331 ATOMIC_CRITICAL_CPT( cmplx10, sub_cpt, kmp_cmplx80, -, 20c, 1 )
2332 ATOMIC_CRITICAL_CPT( cmplx10, mul_cpt, kmp_cmplx80, *, 20c, 1 )
2333 ATOMIC_CRITICAL_CPT( cmplx10, div_cpt, kmp_cmplx80, /, 20c, 1 )
2335 ATOMIC_CRITICAL_CPT( cmplx16, add_cpt, CPLX128_LEG, +, 32c, 1 )
2336 ATOMIC_CRITICAL_CPT( cmplx16, sub_cpt, CPLX128_LEG, -, 32c, 1 )
2337 ATOMIC_CRITICAL_CPT( cmplx16, mul_cpt, CPLX128_LEG, *, 32c, 1 )
2338 ATOMIC_CRITICAL_CPT( cmplx16, div_cpt, CPLX128_LEG, /, 32c, 1 )
2339 #if ( KMP_ARCH_X86 ) 2340 ATOMIC_CRITICAL_CPT( cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c, 1 )
2341 ATOMIC_CRITICAL_CPT( cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c, 1 )
2342 ATOMIC_CRITICAL_CPT( cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c, 1 )
2343 ATOMIC_CRITICAL_CPT( cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c, 1 )
2358 #define OP_CRITICAL_CPT_REV(OP,LCK_ID) \ 2359 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2363 (*lhs) = (rhs) OP (*lhs); \ 2364 new_value = (*lhs); \ 2366 new_value = (*lhs);\ 2367 (*lhs) = (rhs) OP (*lhs); \ 2369 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2373 #ifdef KMP_GOMP_COMPAT 2374 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) \ 2375 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2377 OP_CRITICAL_CPT_REV( OP, 0 ); \ 2380 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) 2390 #define OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \ 2392 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2393 TYPE old_value, new_value; \ 2395 old_value = temp_val; \ 2396 new_value = rhs OP old_value; \ 2397 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 2398 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 2399 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 2404 old_value = temp_val; \ 2405 new_value = rhs OP old_value; \ 2414 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 2415 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2417 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2418 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \ 2419 OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \ 2423 ATOMIC_CMPXCHG_CPT_REV( fixed1, div_cpt_rev, kmp_int8, 8, /, KMP_ARCH_X86 )
2424 ATOMIC_CMPXCHG_CPT_REV( fixed1u, div_cpt_rev, kmp_uint8, 8, /, KMP_ARCH_X86 )
2425 ATOMIC_CMPXCHG_CPT_REV( fixed1, shl_cpt_rev, kmp_int8, 8, <<, KMP_ARCH_X86 )
2426 ATOMIC_CMPXCHG_CPT_REV( fixed1, shr_cpt_rev, kmp_int8, 8, >>, KMP_ARCH_X86 )
2427 ATOMIC_CMPXCHG_CPT_REV( fixed1u, shr_cpt_rev, kmp_uint8, 8, >>, KMP_ARCH_X86 )
2428 ATOMIC_CMPXCHG_CPT_REV( fixed1, sub_cpt_rev, kmp_int8, 8, -, KMP_ARCH_X86 )
2429 ATOMIC_CMPXCHG_CPT_REV( fixed2, div_cpt_rev, kmp_int16, 16, /, KMP_ARCH_X86 )
2430 ATOMIC_CMPXCHG_CPT_REV( fixed2u, div_cpt_rev, kmp_uint16, 16, /, KMP_ARCH_X86 )
2431 ATOMIC_CMPXCHG_CPT_REV( fixed2, shl_cpt_rev, kmp_int16, 16, <<, KMP_ARCH_X86 )
2432 ATOMIC_CMPXCHG_CPT_REV( fixed2, shr_cpt_rev, kmp_int16, 16, >>, KMP_ARCH_X86 )
2433 ATOMIC_CMPXCHG_CPT_REV( fixed2u, shr_cpt_rev, kmp_uint16, 16, >>, KMP_ARCH_X86 )
2434 ATOMIC_CMPXCHG_CPT_REV( fixed2, sub_cpt_rev, kmp_int16, 16, -, KMP_ARCH_X86 )
2435 ATOMIC_CMPXCHG_CPT_REV( fixed4, div_cpt_rev, kmp_int32, 32, /, KMP_ARCH_X86 )
2436 ATOMIC_CMPXCHG_CPT_REV( fixed4u, div_cpt_rev, kmp_uint32, 32, /, KMP_ARCH_X86 )
2437 ATOMIC_CMPXCHG_CPT_REV( fixed4, shl_cpt_rev, kmp_int32, 32, <<, KMP_ARCH_X86 )
2438 ATOMIC_CMPXCHG_CPT_REV( fixed4, shr_cpt_rev, kmp_int32, 32, >>, KMP_ARCH_X86 )
2439 ATOMIC_CMPXCHG_CPT_REV( fixed4u, shr_cpt_rev, kmp_uint32, 32, >>, KMP_ARCH_X86 )
2440 ATOMIC_CMPXCHG_CPT_REV( fixed4, sub_cpt_rev, kmp_int32, 32, -, KMP_ARCH_X86 )
2441 ATOMIC_CMPXCHG_CPT_REV( fixed8, div_cpt_rev, kmp_int64, 64, /, KMP_ARCH_X86 )
2442 ATOMIC_CMPXCHG_CPT_REV( fixed8u, div_cpt_rev, kmp_uint64, 64, /, KMP_ARCH_X86 )
2443 ATOMIC_CMPXCHG_CPT_REV( fixed8, shl_cpt_rev, kmp_int64, 64, <<, KMP_ARCH_X86 )
2444 ATOMIC_CMPXCHG_CPT_REV( fixed8, shr_cpt_rev, kmp_int64, 64, >>, KMP_ARCH_X86 )
2445 ATOMIC_CMPXCHG_CPT_REV( fixed8u, shr_cpt_rev, kmp_uint64, 64, >>, KMP_ARCH_X86 )
2446 ATOMIC_CMPXCHG_CPT_REV( fixed8, sub_cpt_rev, kmp_int64, 64, -, KMP_ARCH_X86 )
2447 ATOMIC_CMPXCHG_CPT_REV( float4, div_cpt_rev, kmp_real32, 32, /, KMP_ARCH_X86 )
2448 ATOMIC_CMPXCHG_CPT_REV( float4, sub_cpt_rev, kmp_real32, 32, -, KMP_ARCH_X86 )
2449 ATOMIC_CMPXCHG_CPT_REV( float8, div_cpt_rev, kmp_real64, 64, /, KMP_ARCH_X86 )
2450 ATOMIC_CMPXCHG_CPT_REV( float8, sub_cpt_rev, kmp_real64, 64, -, KMP_ARCH_X86 )
2459 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2460 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2462 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2464 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \ 2465 OP_CRITICAL_CPT_REV(OP,LCK_ID) \ 2471 ATOMIC_CRITICAL_CPT_REV( float10, sub_cpt_rev,
long double, -, 10r, 1 )
2472 ATOMIC_CRITICAL_CPT_REV( float10, div_cpt_rev,
long double, /, 10r, 1 )
2475 ATOMIC_CRITICAL_CPT_REV( float16, sub_cpt_rev, QUAD_LEGACY, -, 16r, 1 )
2476 ATOMIC_CRITICAL_CPT_REV( float16, div_cpt_rev, QUAD_LEGACY, /, 16r, 1 )
2477 #if ( KMP_ARCH_X86 ) 2478 ATOMIC_CRITICAL_CPT_REV( float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r, 1 )
2479 ATOMIC_CRITICAL_CPT_REV( float16, div_a16_cpt_rev, Quad_a16_t, /, 16r, 1 )
2489 #define OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \ 2490 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2493 (*lhs) = (rhs) OP (*lhs); \ 2497 (*lhs) = (rhs) OP (*lhs); \ 2500 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2504 #ifdef KMP_GOMP_COMPAT 2505 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) \ 2506 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2508 OP_CRITICAL_CPT_REV_WRK( OP, 0 ); \ 2511 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) 2515 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2516 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \ 2517 OP_GOMP_CRITICAL_CPT_REV_WRK(OP,GOMP_FLAG) \ 2518 OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \ 2525 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c, 1 )
2526 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c, 1 )
2528 ATOMIC_CRITICAL_CPT_REV( cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c, 1 )
2529 ATOMIC_CRITICAL_CPT_REV( cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c, 1 )
2530 ATOMIC_CRITICAL_CPT_REV( cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c, 1 )
2531 ATOMIC_CRITICAL_CPT_REV( cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c, 1 )
2533 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c, 1 )
2534 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c, 1 )
2535 #if ( KMP_ARCH_X86 ) 2536 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c, 1 )
2537 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c, 1 )
2549 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 2550 ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \ 2552 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \ 2553 OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \ 2557 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \ 2558 ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \ 2560 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \ 2561 OP_CRITICAL_CPT_REV(OP,LCK_ID) \ 2564 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2565 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2566 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2567 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
2569 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2570 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2571 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2572 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
2574 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i, 3, 0 )
2575 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad, 4i, 3, 0 )
2576 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i, 3, 0 )
2577 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad, 4i, 3, 0 )
2579 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2580 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2581 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2582 ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
2584 ATOMIC_CMPXCHG_CPT_REV_MIX( float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
2585 ATOMIC_CMPXCHG_CPT_REV_MIX( float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
2587 ATOMIC_CMPXCHG_CPT_REV_MIX( float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
2588 ATOMIC_CMPXCHG_CPT_REV_MIX( float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
2590 ATOMIC_CRITICAL_CPT_REV_MIX( float10,
long double, sub_cpt_rev, -, fp, _Quad, 10r, 1 )
2591 ATOMIC_CRITICAL_CPT_REV_MIX( float10,
long double, div_cpt_rev, /, fp, _Quad, 10r, 1 )
2593 #endif //KMP_HAVE_QUAD 2598 #define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2599 TYPE __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \ 2601 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 2602 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid )); 2604 #define CRITICAL_SWP(LCK_ID) \ 2605 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2607 old_value = (*lhs); \ 2610 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2614 #ifdef KMP_GOMP_COMPAT 2615 #define GOMP_CRITICAL_SWP(FLAG) \ 2616 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2618 CRITICAL_SWP( 0 ); \ 2621 #define GOMP_CRITICAL_SWP(FLAG) 2625 #define ATOMIC_XCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \ 2626 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2628 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2629 old_value = KMP_XCHG_FIXED##BITS( lhs, rhs ); \ 2633 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \ 2634 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2636 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2637 old_value = KMP_XCHG_REAL##BITS( lhs, rhs ); \ 2642 #define CMPXCHG_SWP(TYPE,BITS) \ 2644 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2645 TYPE old_value, new_value; \ 2647 old_value = temp_val; \ 2649 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 2650 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 2651 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 2656 old_value = temp_val; \ 2663 #define ATOMIC_CMPXCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \ 2664 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2666 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2667 CMPXCHG_SWP(TYPE,BITS) \ 2670 ATOMIC_XCHG_SWP( fixed1, kmp_int8, 8, KMP_ARCH_X86 )
2671 ATOMIC_XCHG_SWP( fixed2, kmp_int16, 16, KMP_ARCH_X86 )
2672 ATOMIC_XCHG_SWP( fixed4, kmp_int32, 32, KMP_ARCH_X86 )
2674 ATOMIC_XCHG_FLOAT_SWP( float4, kmp_real32, 32, KMP_ARCH_X86 )
2676 #if ( KMP_ARCH_X86 ) 2677 ATOMIC_CMPXCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2678 ATOMIC_CMPXCHG_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2680 ATOMIC_XCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2681 ATOMIC_XCHG_FLOAT_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2686 #define ATOMIC_CRITICAL_SWP(TYPE_ID,TYPE,LCK_ID,GOMP_FLAG) \ 2687 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2689 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2690 CRITICAL_SWP(LCK_ID) \ 2699 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \ 2700 void __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out ) \ 2702 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 2703 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid )); 2706 #define CRITICAL_SWP_WRK(LCK_ID) \ 2707 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2712 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2717 #ifdef KMP_GOMP_COMPAT 2718 #define GOMP_CRITICAL_SWP_WRK(FLAG) \ 2719 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2721 CRITICAL_SWP_WRK( 0 ); \ 2724 #define GOMP_CRITICAL_SWP_WRK(FLAG) 2728 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE,LCK_ID,GOMP_FLAG) \ 2729 ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \ 2731 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \ 2732 CRITICAL_SWP_WRK(LCK_ID) \ 2737 ATOMIC_CRITICAL_SWP( float10,
long double, 10r, 1 )
2739 ATOMIC_CRITICAL_SWP( float16, QUAD_LEGACY, 16r, 1 )
2742 ATOMIC_CRITICAL_SWP_WRK( cmplx4, kmp_cmplx32, 8c, 1 )
2747 ATOMIC_CRITICAL_SWP( cmplx8, kmp_cmplx64, 16c, 1 )
2748 ATOMIC_CRITICAL_SWP( cmplx10, kmp_cmplx80, 20c, 1 )
2750 ATOMIC_CRITICAL_SWP( cmplx16, CPLX128_LEG, 32c, 1 )
2751 #if ( KMP_ARCH_X86 ) 2752 ATOMIC_CRITICAL_SWP( float16_a16, Quad_a16_t, 16r, 1 )
2753 ATOMIC_CRITICAL_SWP( cmplx16_a16, kmp_cmplx128_a16_t, 32c, 1 )
2760 #endif //OMP_40_ENABLED 2762 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64 2772 __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2774 KMP_DEBUG_ASSERT( __kmp_init_serial );
2777 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2784 kmp_int8 old_value, new_value;
2786 old_value = *(kmp_int8 *) lhs;
2787 (*f)( &new_value, &old_value, rhs );
2790 while ( ! KMP_COMPARE_AND_STORE_ACQ8 ( (kmp_int8 *) lhs,
2791 *(kmp_int8 *) &old_value, *(kmp_int8 *) &new_value ) )
2795 old_value = *(kmp_int8 *) lhs;
2796 (*f)( &new_value, &old_value, rhs );
2806 #ifdef KMP_GOMP_COMPAT 2807 if ( __kmp_atomic_mode == 2 ) {
2808 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2812 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2814 (*f)( lhs, lhs, rhs );
2816 #ifdef KMP_GOMP_COMPAT 2817 if ( __kmp_atomic_mode == 2 ) {
2818 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2822 __kmp_release_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2827 __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2830 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2832 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2835 ! ( (kmp_uintptr_t) lhs & 0x1)
2839 kmp_int16 old_value, new_value;
2841 old_value = *(kmp_int16 *) lhs;
2842 (*f)( &new_value, &old_value, rhs );
2845 while ( ! KMP_COMPARE_AND_STORE_ACQ16 ( (kmp_int16 *) lhs,
2846 *(kmp_int16 *) &old_value, *(kmp_int16 *) &new_value ) )
2850 old_value = *(kmp_int16 *) lhs;
2851 (*f)( &new_value, &old_value, rhs );
2861 #ifdef KMP_GOMP_COMPAT 2862 if ( __kmp_atomic_mode == 2 ) {
2863 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2867 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2869 (*f)( lhs, lhs, rhs );
2871 #ifdef KMP_GOMP_COMPAT 2872 if ( __kmp_atomic_mode == 2 ) {
2873 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2877 __kmp_release_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2882 __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2884 KMP_DEBUG_ASSERT( __kmp_init_serial );
2891 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
2894 ! ( (kmp_uintptr_t) lhs & 0x3)
2898 kmp_int32 old_value, new_value;
2900 old_value = *(kmp_int32 *) lhs;
2901 (*f)( &new_value, &old_value, rhs );
2904 while ( ! KMP_COMPARE_AND_STORE_ACQ32 ( (kmp_int32 *) lhs,
2905 *(kmp_int32 *) &old_value, *(kmp_int32 *) &new_value ) )
2909 old_value = *(kmp_int32 *) lhs;
2910 (*f)( &new_value, &old_value, rhs );
2921 #ifdef KMP_GOMP_COMPAT 2922 if ( __kmp_atomic_mode == 2 ) {
2923 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2927 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2929 (*f)( lhs, lhs, rhs );
2931 #ifdef KMP_GOMP_COMPAT 2932 if ( __kmp_atomic_mode == 2 ) {
2933 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2937 __kmp_release_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2942 __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2944 KMP_DEBUG_ASSERT( __kmp_init_serial );
2947 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2949 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2952 ! ( (kmp_uintptr_t) lhs & 0x7)
2956 kmp_int64 old_value, new_value;
2958 old_value = *(kmp_int64 *) lhs;
2959 (*f)( &new_value, &old_value, rhs );
2961 while ( ! KMP_COMPARE_AND_STORE_ACQ64 ( (kmp_int64 *) lhs,
2962 *(kmp_int64 *) &old_value,
2963 *(kmp_int64 *) &new_value ) )
2967 old_value = *(kmp_int64 *) lhs;
2968 (*f)( &new_value, &old_value, rhs );
2978 #ifdef KMP_GOMP_COMPAT 2979 if ( __kmp_atomic_mode == 2 ) {
2980 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2984 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2986 (*f)( lhs, lhs, rhs );
2988 #ifdef KMP_GOMP_COMPAT 2989 if ( __kmp_atomic_mode == 2 ) {
2990 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2994 __kmp_release_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2999 __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
3001 KMP_DEBUG_ASSERT( __kmp_init_serial );
3003 #ifdef KMP_GOMP_COMPAT 3004 if ( __kmp_atomic_mode == 2 ) {
3005 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
3009 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_10r, gtid );
3011 (*f)( lhs, lhs, rhs );
3013 #ifdef KMP_GOMP_COMPAT 3014 if ( __kmp_atomic_mode == 2 ) {
3015 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
3019 __kmp_release_atomic_lock( & __kmp_atomic_lock_10r, gtid );
3023 __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
3025 KMP_DEBUG_ASSERT( __kmp_init_serial );
3027 #ifdef KMP_GOMP_COMPAT 3028 if ( __kmp_atomic_mode == 2 ) {
3029 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
3033 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_16c, gtid );
3035 (*f)( lhs, lhs, rhs );
3037 #ifdef KMP_GOMP_COMPAT 3038 if ( __kmp_atomic_mode == 2 ) {
3039 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
3043 __kmp_release_atomic_lock( & __kmp_atomic_lock_16c, gtid );
3047 __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
3049 KMP_DEBUG_ASSERT( __kmp_init_serial );
3051 #ifdef KMP_GOMP_COMPAT 3052 if ( __kmp_atomic_mode == 2 ) {
3053 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
3057 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_20c, gtid );
3059 (*f)( lhs, lhs, rhs );
3061 #ifdef KMP_GOMP_COMPAT 3062 if ( __kmp_atomic_mode == 2 ) {
3063 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
3067 __kmp_release_atomic_lock( & __kmp_atomic_lock_20c, gtid );
3071 __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
3073 KMP_DEBUG_ASSERT( __kmp_init_serial );
3075 #ifdef KMP_GOMP_COMPAT 3076 if ( __kmp_atomic_mode == 2 ) {
3077 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
3081 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_32c, gtid );
3083 (*f)( lhs, lhs, rhs );
3085 #ifdef KMP_GOMP_COMPAT 3086 if ( __kmp_atomic_mode == 2 ) {
3087 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
3091 __kmp_release_atomic_lock( & __kmp_atomic_lock_32c, gtid );
3098 __kmpc_atomic_start(
void)
3100 int gtid = __kmp_entry_gtid();
3101 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3102 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3107 __kmpc_atomic_end(
void)
3109 int gtid = __kmp_get_gtid();
3110 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3111 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);