13 #include "kmp_atomic.h" 16 typedef unsigned char uchar;
17 typedef unsigned short ushort;
565 #ifndef KMP_GOMP_COMPAT 566 int __kmp_atomic_mode = 1;
568 int __kmp_atomic_mode = 2;
574 kmp_atomic_lock_t __kmp_atomic_lock;
576 kmp_atomic_lock_t __kmp_atomic_lock_1i;
578 kmp_atomic_lock_t __kmp_atomic_lock_2i;
580 kmp_atomic_lock_t __kmp_atomic_lock_4i;
582 kmp_atomic_lock_t __kmp_atomic_lock_4r;
584 kmp_atomic_lock_t __kmp_atomic_lock_8i;
586 kmp_atomic_lock_t __kmp_atomic_lock_8r;
588 kmp_atomic_lock_t __kmp_atomic_lock_8c;
590 kmp_atomic_lock_t __kmp_atomic_lock_10r;
592 kmp_atomic_lock_t __kmp_atomic_lock_16r;
594 kmp_atomic_lock_t __kmp_atomic_lock_16c;
596 kmp_atomic_lock_t __kmp_atomic_lock_20c;
598 kmp_atomic_lock_t __kmp_atomic_lock_32c;
605 #define KMP_ATOMIC_VOLATILE volatile 607 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD 609 static inline Quad_a4_t operator+(Quad_a4_t &lhs, Quad_a4_t &rhs) {
610 return lhs.q + rhs.q;
612 static inline Quad_a4_t operator-(Quad_a4_t &lhs, Quad_a4_t &rhs) {
613 return lhs.q - rhs.q;
615 static inline Quad_a4_t operator*(Quad_a4_t &lhs, Quad_a4_t &rhs) {
616 return lhs.q * rhs.q;
618 static inline Quad_a4_t operator/(Quad_a4_t &lhs, Quad_a4_t &rhs) {
619 return lhs.q / rhs.q;
621 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 return lhs.q < rhs.q;
624 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q > rhs.q;
628 static inline Quad_a16_t operator+(Quad_a16_t &lhs, Quad_a16_t &rhs) {
629 return lhs.q + rhs.q;
631 static inline Quad_a16_t operator-(Quad_a16_t &lhs, Quad_a16_t &rhs) {
632 return lhs.q - rhs.q;
634 static inline Quad_a16_t operator*(Quad_a16_t &lhs, Quad_a16_t &rhs) {
635 return lhs.q * rhs.q;
637 static inline Quad_a16_t operator/(Quad_a16_t &lhs, Quad_a16_t &rhs) {
638 return lhs.q / rhs.q;
640 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 return lhs.q < rhs.q;
643 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q > rhs.q;
647 static inline kmp_cmplx128_a4_t operator+(kmp_cmplx128_a4_t &lhs,
648 kmp_cmplx128_a4_t &rhs) {
649 return lhs.q + rhs.q;
651 static inline kmp_cmplx128_a4_t operator-(kmp_cmplx128_a4_t &lhs,
652 kmp_cmplx128_a4_t &rhs) {
653 return lhs.q - rhs.q;
655 static inline kmp_cmplx128_a4_t operator*(kmp_cmplx128_a4_t &lhs,
656 kmp_cmplx128_a4_t &rhs) {
657 return lhs.q * rhs.q;
659 static inline kmp_cmplx128_a4_t operator/(kmp_cmplx128_a4_t &lhs,
660 kmp_cmplx128_a4_t &rhs) {
661 return lhs.q / rhs.q;
664 static inline kmp_cmplx128_a16_t operator+(kmp_cmplx128_a16_t &lhs,
665 kmp_cmplx128_a16_t &rhs) {
666 return lhs.q + rhs.q;
668 static inline kmp_cmplx128_a16_t operator-(kmp_cmplx128_a16_t &lhs,
669 kmp_cmplx128_a16_t &rhs) {
670 return lhs.q - rhs.q;
672 static inline kmp_cmplx128_a16_t operator*(kmp_cmplx128_a16_t &lhs,
673 kmp_cmplx128_a16_t &rhs) {
674 return lhs.q * rhs.q;
676 static inline kmp_cmplx128_a16_t operator/(kmp_cmplx128_a16_t &lhs,
677 kmp_cmplx128_a16_t &rhs) {
678 return lhs.q / rhs.q;
681 #endif // (KMP_ARCH_X86) && KMP_HAVE_QUAD 688 #define KMP_CHECK_GTID \ 689 if (gtid == KMP_GTID_UNKNOWN) { \ 690 gtid = __kmp_entry_gtid(); \ 691 } // check and get gtid when needed 698 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 699 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 700 TYPE *lhs, TYPE rhs) { \ 701 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 702 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 706 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat 707 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char 708 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short 709 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int 710 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float 711 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int 712 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double 713 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex 714 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double 715 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad 716 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex 717 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex 718 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex 726 #define OP_CRITICAL(OP, LCK_ID) \ 727 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 731 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 733 #define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \ 734 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 735 (*lhs) = (TYPE)((*lhs)OP((TYPE)rhs)); \ 736 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 760 #ifdef KMP_GOMP_COMPAT 761 #define OP_GOMP_CRITICAL(OP, FLAG) \ 762 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 764 OP_CRITICAL(OP, 0); \ 768 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) \ 769 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 771 OP_UPDATE_CRITICAL(TYPE, OP, 0); \ 775 #define OP_GOMP_CRITICAL(OP, FLAG) 776 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) 780 #define KMP_DO_PAUSE _mm_delay_32(1) 782 #define KMP_DO_PAUSE KMP_CPU_PAUSE() 790 #define OP_CMPXCHG(TYPE, BITS, OP) \ 792 TYPE old_value, new_value; \ 793 old_value = *(TYPE volatile *)lhs; \ 794 new_value = (TYPE)(old_value OP((TYPE)rhs)); \ 795 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 796 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 797 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 800 old_value = *(TYPE volatile *)lhs; \ 801 new_value = (TYPE)(old_value OP((TYPE)rhs)); \ 812 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 816 kmp_int##BITS *vvv; \ 818 struct _sss old_value, new_value; \ 819 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \ 820 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \ 821 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 822 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \ 823 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 824 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ 825 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \ 828 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 829 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \ 833 #endif // USE_CMPXCHG_FIX 835 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 839 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 841 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 842 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 844 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 847 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 849 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 850 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 851 OP_CMPXCHG(TYPE, BITS, OP) \ 856 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 858 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 859 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 860 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 863 #endif // USE_CMPXCHG_FIX 868 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 870 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 871 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 872 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 874 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 877 OP_UPDATE_CRITICAL(TYPE, OP, \ 882 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 884 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 885 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 886 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 887 OP_CMPXCHG(TYPE, BITS, OP) \ 890 OP_UPDATE_CRITICAL(TYPE, OP, \ 897 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 899 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 900 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 901 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 902 OP_CMPXCHG(TYPE, BITS, OP) \ 905 OP_UPDATE_CRITICAL(TYPE, OP, \ 910 #endif // USE_CMPXCHG_FIX 914 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
916 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
919 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
921 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
925 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
927 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
930 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
932 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
950 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
952 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
954 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
956 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
958 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
960 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
962 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
964 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
966 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
968 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
970 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
972 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
974 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
976 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
978 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
980 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
982 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
984 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
986 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
988 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
990 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
992 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
994 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
996 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
998 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
1000 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
1002 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
1004 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
1006 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
1008 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
1010 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
1012 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
1014 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
1016 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
1018 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1020 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1022 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1024 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1026 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1028 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1030 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1032 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1034 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1036 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1046 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1047 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1048 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1049 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1052 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1056 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1057 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1058 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1059 OP_CMPXCHG(TYPE, BITS, OP) \ 1065 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1066 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1067 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1068 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1069 OP_CMPXCHG(TYPE, BITS, OP) \ 1072 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1077 ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1079 ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1081 ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1083 ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1085 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1087 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1089 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1091 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1103 #define MIN_MAX_CRITSECT(OP, LCK_ID) \ 1104 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1106 if (*lhs OP rhs) { \ 1109 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1112 #ifdef KMP_GOMP_COMPAT 1113 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \ 1114 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1116 MIN_MAX_CRITSECT(OP, 0); \ 1120 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) 1124 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1126 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1129 old_value = temp_val; \ 1130 while (old_value OP rhs && \ 1131 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1132 (kmp_int##BITS *)lhs, \ 1133 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1134 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 1137 old_value = temp_val; \ 1143 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1144 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1145 if (*lhs OP rhs) { \ 1146 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1147 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1151 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1155 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1157 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1158 if (*lhs OP rhs) { \ 1159 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1160 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1167 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1169 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1170 if (*lhs OP rhs) { \ 1171 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1172 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1173 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1176 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1182 MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1184 MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1186 MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1188 MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1190 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1192 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1194 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1196 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1198 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1200 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1202 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1204 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1207 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1209 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1212 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1214 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1216 #endif // (KMP_ARCH_X86) 1217 #endif // KMP_HAVE_QUAD 1221 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1222 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1223 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \ 1224 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \ 1228 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1231 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1233 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1234 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \ 1235 OP_CMPXCHG(TYPE, BITS, OP) \ 1241 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1243 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1244 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \ 1245 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1246 OP_CMPXCHG(TYPE, BITS, OP) \ 1249 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \ 1254 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1256 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1258 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1260 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1262 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1264 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1266 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1268 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1277 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1278 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1279 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1280 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \ 1285 ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1287 ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1289 ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1291 ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1295 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1297 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1299 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1301 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1304 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1306 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1308 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1310 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1312 #endif // (KMP_ARCH_X86) 1313 #endif // KMP_HAVE_QUAD 1318 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1320 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1322 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1324 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1328 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1329 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1330 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1331 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1332 #endif // USE_CMPXCHG_FIX 1334 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1335 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1336 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1337 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1338 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1340 ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1342 ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1344 ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1347 ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1349 ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1351 ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1353 ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1356 ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1358 ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1360 ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1362 ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1364 #endif // (KMP_ARCH_X86) 1365 #endif // KMP_HAVE_QUAD 1369 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1377 #define OP_CRITICAL_REV(TYPE, OP, LCK_ID) \ 1378 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1380 (*lhs) = (TYPE)((rhs)OP(*lhs)); \ 1382 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1384 #ifdef KMP_GOMP_COMPAT 1385 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) \ 1386 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1388 OP_CRITICAL_REV(TYPE, OP, 0); \ 1393 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) 1401 #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1402 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \ 1403 TYPE *lhs, TYPE rhs) { \ 1404 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1405 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid)); 1414 #define OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1416 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1417 TYPE old_value, new_value; \ 1419 old_value = temp_val; \ 1420 new_value = (TYPE)(rhs OP old_value); \ 1421 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1422 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1423 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 1427 old_value = temp_val; \ 1428 new_value = (TYPE)(rhs OP old_value); \ 1433 #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \ 1434 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1435 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1436 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1453 ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1455 ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1457 ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1459 ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1461 ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1463 ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1466 ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1468 ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1470 ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1472 ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1474 ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1476 ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1479 ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1481 ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1483 ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1485 ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1487 ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1489 ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1492 ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1494 ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1496 ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1498 ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1500 ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1502 ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1505 ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1507 ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1510 ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1512 ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1522 #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1523 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1524 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1525 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \ 1530 ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1532 ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1536 ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1538 ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1541 ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1543 ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1545 #endif // KMP_ARCH_X86 1546 #endif // KMP_HAVE_QUAD 1549 ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1551 ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1553 ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1555 ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1557 ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1559 ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1562 ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1564 ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1567 ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1569 ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1571 #endif // KMP_ARCH_X86 1572 #endif // KMP_HAVE_QUAD 1574 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 1588 #define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1589 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 1590 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \ 1591 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1593 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 1597 #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \ 1599 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1600 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1601 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \ 1605 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1608 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1609 LCK_ID, MASK, GOMP_FLAG) \ 1610 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1611 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1612 OP_CMPXCHG(TYPE, BITS, OP) \ 1618 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1619 LCK_ID, MASK, GOMP_FLAG) \ 1620 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1621 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1622 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1623 OP_CMPXCHG(TYPE, BITS, OP) \ 1626 OP_UPDATE_CRITICAL(TYPE, OP, \ 1633 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1635 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 1636 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 1637 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1638 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1639 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1641 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 1642 LCK_ID, GOMP_FLAG) \ 1643 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1644 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \ 1645 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \ 1650 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1652 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1654 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1656 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1658 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1660 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1662 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1664 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1666 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1668 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1670 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1672 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1678 ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1680 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1682 ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1684 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1686 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1688 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1690 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1692 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1695 ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1697 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1699 ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1701 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1703 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1705 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1707 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1709 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1712 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1714 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1716 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1718 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1720 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1722 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1724 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1726 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1729 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1731 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1733 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1735 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1737 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1739 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1741 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1743 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1746 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1748 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1750 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1752 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1755 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1757 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1759 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1761 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1764 ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1766 ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1768 ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1770 ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1773 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1775 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1777 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1779 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1781 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1784 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1786 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1788 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1790 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1793 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1795 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1797 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1799 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1802 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1804 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1806 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1808 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1811 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1813 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1816 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1818 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1821 ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1823 ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1827 #endif // KMP_HAVE_QUAD 1829 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1834 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1835 LCK_ID, MASK, GOMP_FLAG) \ 1836 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1837 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1838 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 1842 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1843 LCK_ID, MASK, GOMP_FLAG) \ 1844 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1845 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1846 OP_CMPXCHG(TYPE, BITS, OP) \ 1848 #endif // USE_CMPXCHG_FIX 1852 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1853 LCK_ID, MASK, GOMP_FLAG) \ 1854 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1855 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \ 1856 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1857 OP_CMPXCHG(TYPE, BITS, OP) \ 1860 OP_UPDATE_CRITICAL(TYPE, OP, \ 1866 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1868 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1870 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1872 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1876 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1887 #define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1888 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 1890 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1891 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 1902 #define OP_CMPXCHG_READ(TYPE, BITS, OP) \ 1904 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1907 kmp_int##BITS i_val; \ 1909 union f_i_union old_value; \ 1911 old_value.f_val = temp_val; \ 1912 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \ 1913 (kmp_int##BITS *)loc, \ 1914 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \ 1915 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \ 1916 new_value = old_value.f_val; \ 1926 #define OP_CRITICAL_READ(OP, LCK_ID) \ 1927 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1929 new_value = (*loc); \ 1931 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1934 #ifdef KMP_GOMP_COMPAT 1935 #define OP_GOMP_CRITICAL_READ(OP, FLAG) \ 1936 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1938 OP_CRITICAL_READ(OP, 0); \ 1942 #define OP_GOMP_CRITICAL_READ(OP, FLAG) 1946 #define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1947 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1949 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1950 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \ 1954 #define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1955 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1957 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1958 OP_CMPXCHG_READ(TYPE, BITS, OP) \ 1966 #define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1967 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1969 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1970 OP_CRITICAL_READ(OP, LCK_ID) \ 1978 #if (KMP_OS_WINDOWS) 1980 #define OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 1981 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1985 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1987 #ifdef KMP_GOMP_COMPAT 1988 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \ 1989 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1991 OP_CRITICAL_READ_WRK(OP, 0); \ 1994 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) 1997 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 1998 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \ 2000 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2001 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2004 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2005 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 2006 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \ 2007 OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 2010 #endif // KMP_OS_WINDOWS 2014 ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
2015 ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
2017 ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
2019 ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2023 ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2025 ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2028 ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2031 ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2033 #endif // KMP_HAVE_QUAD 2036 #if (KMP_OS_WINDOWS) 2037 ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2040 ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2042 #endif // (KMP_OS_WINDOWS) 2043 ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2045 ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2048 ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2051 ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2053 ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2055 #endif // (KMP_ARCH_X86) 2056 #endif // KMP_HAVE_QUAD 2061 #define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2062 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2063 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2064 KMP_XCHG_FIXED##BITS(lhs, rhs); \ 2067 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2068 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2069 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2070 KMP_XCHG_REAL##BITS(lhs, rhs); \ 2080 #define OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2082 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2083 TYPE old_value, new_value; \ 2085 old_value = temp_val; \ 2087 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2088 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2089 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2093 old_value = temp_val; \ 2099 #define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2100 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2101 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2102 OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2111 #define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2112 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2113 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2114 OP_CRITICAL(OP, LCK_ID) \ 2118 ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2120 ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2122 ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2125 ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2128 ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2130 #endif // (KMP_ARCH_X86) 2132 ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2135 ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2138 ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2140 #endif // (KMP_ARCH_X86) 2142 ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2145 ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2147 #endif // KMP_HAVE_QUAD 2148 ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2149 ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2151 ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2154 ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2157 ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2159 ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2161 #endif // (KMP_ARCH_X86) 2162 #endif // KMP_HAVE_QUAD 2172 #define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 2173 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 2174 TYPE *lhs, TYPE rhs, int flag) { \ 2175 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2176 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2184 #define OP_CRITICAL_CPT(OP, LCK_ID) \ 2185 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2189 new_value = (*lhs); \ 2191 new_value = (*lhs); \ 2195 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2198 #define OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \ 2199 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2202 (*lhs) = (TYPE)((*lhs)OP rhs); \ 2203 new_value = (*lhs); \ 2205 new_value = (*lhs); \ 2206 (*lhs) = (TYPE)((*lhs)OP rhs); \ 2209 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2213 #ifdef KMP_GOMP_COMPAT 2214 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) \ 2215 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2217 OP_UPDATE_CRITICAL_CPT(TYPE, OP, 0); \ 2220 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) 2230 #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2232 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2233 TYPE old_value, new_value; \ 2235 old_value = temp_val; \ 2236 new_value = (TYPE)(old_value OP rhs); \ 2237 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2238 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2239 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2243 old_value = temp_val; \ 2244 new_value = (TYPE)(old_value OP rhs); \ 2253 #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2254 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2256 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2257 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2261 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2262 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2263 TYPE old_value, new_value; \ 2264 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2266 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 2268 return old_value OP rhs; \ 2274 ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2276 ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2278 ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2280 ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2283 ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2285 ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2287 ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2289 ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2304 ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2306 ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2308 ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2310 ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2312 ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2314 ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2316 ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2318 ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2320 ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2322 ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2324 ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2326 ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2328 ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2330 ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2332 ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2334 ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2336 ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2338 ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2340 ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2342 ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2344 ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2346 ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2348 ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2350 ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2352 ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2354 ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2356 ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2358 ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2360 ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2362 ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2364 ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2366 ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2368 ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2370 ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2372 ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2374 ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2376 ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2378 ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2380 ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2382 ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2384 ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2386 ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2388 ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2390 ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2402 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2403 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 2404 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \ 2405 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2407 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 2411 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 2412 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 2413 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2415 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2416 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2420 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 2421 LCK_ID, GOMP_FLAG) \ 2422 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2424 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2425 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \ 2428 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2430 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2432 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2434 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2436 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2438 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2440 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2442 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2445 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2447 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2449 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2451 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2453 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2455 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2457 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2459 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2462 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2464 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2466 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2468 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2470 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2472 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2474 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2476 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2479 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2481 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2483 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2485 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2487 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2489 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2491 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2493 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2496 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2498 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2500 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2502 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2505 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2507 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2509 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2511 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2514 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2516 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2518 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2520 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2523 #endif // KMP_HAVE_QUAD 2534 #define OP_CRITICAL_L_CPT(OP, LCK_ID) \ 2535 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2539 (*lhs) = new_value; \ 2541 new_value = (*lhs); \ 2545 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 2548 #ifdef KMP_GOMP_COMPAT 2549 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \ 2550 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2552 OP_CRITICAL_L_CPT(OP, 0); \ 2556 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) 2561 #define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2562 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2564 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \ 2565 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2568 ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2570 ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2572 ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2574 ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2576 ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2578 ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2580 ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2582 ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2594 #define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2595 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2597 if (*lhs OP rhs) { \ 2603 new_value = old_value; \ 2607 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2611 #ifdef KMP_GOMP_COMPAT 2612 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \ 2613 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2615 MIN_MAX_CRITSECT_CPT(OP, 0); \ 2618 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) 2622 #define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2624 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2627 old_value = temp_val; \ 2628 while (old_value OP rhs && \ 2629 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2630 (kmp_int##BITS *)lhs, \ 2631 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2632 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 2635 old_value = temp_val; \ 2645 #define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2646 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2647 TYPE new_value, old_value; \ 2648 if (*lhs OP rhs) { \ 2649 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2650 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2655 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2656 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2657 TYPE new_value, old_value; \ 2658 if (*lhs OP rhs) { \ 2659 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2660 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2665 MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2667 MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2669 MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2671 MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2673 MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2675 MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2677 MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2679 MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2681 MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2683 MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2685 MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2687 MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2690 MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2692 MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2695 MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2697 MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2699 #endif // (KMP_ARCH_X86) 2700 #endif // KMP_HAVE_QUAD 2703 #ifdef KMP_GOMP_COMPAT 2704 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \ 2705 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2707 OP_CRITICAL_CPT(OP, 0); \ 2710 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) 2713 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2714 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2716 OP_GOMP_CRITICAL_EQV_CPT(^= (TYPE) ~, GOMP_FLAG) \ 2717 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2722 ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2724 ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2726 ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2728 ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2730 ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2732 ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2734 ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2736 ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2745 #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2746 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2748 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \ 2749 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \ 2755 #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \ 2756 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2766 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2770 #ifdef KMP_GOMP_COMPAT 2771 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \ 2772 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2774 OP_CRITICAL_CPT_WRK(OP## =, 0); \ 2777 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) 2781 #define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2782 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \ 2783 TYPE rhs, TYPE *out, int flag) { \ 2784 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2785 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2788 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2789 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2790 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \ 2791 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \ 2797 ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2799 ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2801 ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2803 ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2807 ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2809 ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2811 ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2813 ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2816 ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2818 ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2820 ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2822 ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2824 #endif // (KMP_ARCH_X86) 2825 #endif // KMP_HAVE_QUAD 2830 ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2832 ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2834 ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2836 ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2839 ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2841 ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2843 ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2845 ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2847 ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2849 ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2851 ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2853 ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2856 ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2858 ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2860 ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2862 ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2865 ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2867 ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2869 ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2871 ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2873 #endif // (KMP_ARCH_X86) 2874 #endif // KMP_HAVE_QUAD 2886 #define OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \ 2887 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2891 (*lhs) = (TYPE)((rhs)OP(*lhs)); \ 2892 new_value = (*lhs); \ 2894 new_value = (*lhs); \ 2895 (*lhs) = (TYPE)((rhs)OP(*lhs)); \ 2897 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2901 #ifdef KMP_GOMP_COMPAT 2902 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) \ 2903 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2905 OP_CRITICAL_CPT_REV(TYPE, OP, 0); \ 2908 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) 2918 #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2920 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2921 TYPE old_value, new_value; \ 2923 old_value = temp_val; \ 2924 new_value = (TYPE)(rhs OP old_value); \ 2925 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2926 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2927 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2931 old_value = temp_val; \ 2932 new_value = (TYPE)(rhs OP old_value); \ 2941 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2942 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2944 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 2945 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2948 ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2950 ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2952 ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
2954 ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
2956 ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
2958 ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
2960 ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
2962 ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
2964 ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
2966 ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
2968 ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
2970 ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
2972 ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
2974 ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
2976 ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
2978 ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
2980 ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
2982 ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
2984 ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
2986 ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
2988 ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
2990 ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
2992 ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
2994 ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
2996 ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
2998 ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
3000 ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
3002 ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
3012 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 3013 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 3016 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 3017 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \ 3022 ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
3024 ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
3028 ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
3030 ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
3033 ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
3035 ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
3037 #endif // (KMP_ARCH_X86) 3038 #endif // KMP_HAVE_QUAD 3045 #define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3046 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3049 (*lhs) = (rhs)OP(*lhs); \ 3053 (*lhs) = (rhs)OP(*lhs); \ 3056 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3060 #ifdef KMP_GOMP_COMPAT 3061 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \ 3062 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3064 OP_CRITICAL_CPT_REV_WRK(OP, 0); \ 3067 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) 3071 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \ 3073 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 3074 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \ 3075 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3081 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3083 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3086 ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3088 ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3090 ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3092 ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3095 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3097 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3100 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3102 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3104 #endif // (KMP_ARCH_X86) 3105 #endif // KMP_HAVE_QUAD 3116 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 3117 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 3118 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3120 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 3121 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 3125 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 3126 LCK_ID, GOMP_FLAG) \ 3127 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3129 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \ 3130 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \ 3133 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3135 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3137 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3139 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3142 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3144 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3147 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3149 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3153 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3155 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3157 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3159 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3162 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3165 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3168 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3171 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3175 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3178 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3182 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3185 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3189 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3191 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3194 #endif // KMP_HAVE_QUAD 3198 #define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3199 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3201 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3202 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3204 #define CRITICAL_SWP(LCK_ID) \ 3205 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3207 old_value = (*lhs); \ 3210 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3214 #ifdef KMP_GOMP_COMPAT 3215 #define GOMP_CRITICAL_SWP(FLAG) \ 3216 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3221 #define GOMP_CRITICAL_SWP(FLAG) 3224 #define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3225 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3227 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3228 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \ 3232 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3233 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3235 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3236 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \ 3241 #define CMPXCHG_SWP(TYPE, BITS) \ 3243 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 3244 TYPE old_value, new_value; \ 3246 old_value = temp_val; \ 3248 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 3249 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 3250 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 3254 old_value = temp_val; \ 3261 #define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3262 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3264 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3265 CMPXCHG_SWP(TYPE, BITS) \ 3268 ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3269 ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3270 ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3272 ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3276 ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3278 ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3281 ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3282 ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3284 #endif // (KMP_ARCH_X86) 3289 #define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3290 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3292 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3293 CRITICAL_SWP(LCK_ID) \ 3301 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3302 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3303 TYPE rhs, TYPE *out) { \ 3304 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3305 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3307 #define CRITICAL_SWP_WRK(LCK_ID) \ 3308 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3313 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3317 #ifdef KMP_GOMP_COMPAT 3318 #define GOMP_CRITICAL_SWP_WRK(FLAG) \ 3319 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3321 CRITICAL_SWP_WRK(0); \ 3324 #define GOMP_CRITICAL_SWP_WRK(FLAG) 3328 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3329 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3331 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \ 3332 CRITICAL_SWP_WRK(LCK_ID) \ 3336 ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3338 ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3339 #endif // KMP_HAVE_QUAD 3341 ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3346 ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3347 ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3349 ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3351 ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3353 ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3355 #endif // (KMP_ARCH_X86) 3356 #endif // KMP_HAVE_QUAD 3360 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 3367 void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3368 void (*f)(
void *,
void *,
void *)) {
3369 KMP_DEBUG_ASSERT(__kmp_init_serial);
3372 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3378 kmp_int8 old_value, new_value;
3380 old_value = *(kmp_int8 *)lhs;
3381 (*f)(&new_value, &old_value, rhs);
3384 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3385 *(kmp_int8 *)&new_value)) {
3388 old_value = *(kmp_int8 *)lhs;
3389 (*f)(&new_value, &old_value, rhs);
3396 #ifdef KMP_GOMP_COMPAT 3397 if (__kmp_atomic_mode == 2) {
3398 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3401 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3403 (*f)(lhs, lhs, rhs);
3405 #ifdef KMP_GOMP_COMPAT 3406 if (__kmp_atomic_mode == 2) {
3407 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3410 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3414 void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3415 void (*f)(
void *,
void *,
void *)) {
3417 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3419 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3422 !((kmp_uintptr_t)lhs & 0x1)
3425 kmp_int16 old_value, new_value;
3427 old_value = *(kmp_int16 *)lhs;
3428 (*f)(&new_value, &old_value, rhs);
3431 while (!KMP_COMPARE_AND_STORE_ACQ16(
3432 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3435 old_value = *(kmp_int16 *)lhs;
3436 (*f)(&new_value, &old_value, rhs);
3443 #ifdef KMP_GOMP_COMPAT 3444 if (__kmp_atomic_mode == 2) {
3445 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3448 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3450 (*f)(lhs, lhs, rhs);
3452 #ifdef KMP_GOMP_COMPAT 3453 if (__kmp_atomic_mode == 2) {
3454 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3457 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3461 void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3462 void (*f)(
void *,
void *,
void *)) {
3463 KMP_DEBUG_ASSERT(__kmp_init_serial);
3468 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3471 !((kmp_uintptr_t)lhs & 0x3)
3474 kmp_int32 old_value, new_value;
3476 old_value = *(kmp_int32 *)lhs;
3477 (*f)(&new_value, &old_value, rhs);
3480 while (!KMP_COMPARE_AND_STORE_ACQ32(
3481 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3484 old_value = *(kmp_int32 *)lhs;
3485 (*f)(&new_value, &old_value, rhs);
3493 #ifdef KMP_GOMP_COMPAT 3494 if (__kmp_atomic_mode == 2) {
3495 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3498 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3500 (*f)(lhs, lhs, rhs);
3502 #ifdef KMP_GOMP_COMPAT 3503 if (__kmp_atomic_mode == 2) {
3504 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3507 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3511 void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3512 void (*f)(
void *,
void *,
void *)) {
3513 KMP_DEBUG_ASSERT(__kmp_init_serial);
3516 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3518 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3521 !((kmp_uintptr_t)lhs & 0x7)
3524 kmp_int64 old_value, new_value;
3526 old_value = *(kmp_int64 *)lhs;
3527 (*f)(&new_value, &old_value, rhs);
3529 while (!KMP_COMPARE_AND_STORE_ACQ64(
3530 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3533 old_value = *(kmp_int64 *)lhs;
3534 (*f)(&new_value, &old_value, rhs);
3542 #ifdef KMP_GOMP_COMPAT 3543 if (__kmp_atomic_mode == 2) {
3544 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3547 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3549 (*f)(lhs, lhs, rhs);
3551 #ifdef KMP_GOMP_COMPAT 3552 if (__kmp_atomic_mode == 2) {
3553 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3556 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3560 void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3561 void (*f)(
void *,
void *,
void *)) {
3562 KMP_DEBUG_ASSERT(__kmp_init_serial);
3564 #ifdef KMP_GOMP_COMPAT 3565 if (__kmp_atomic_mode == 2) {
3566 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3569 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3571 (*f)(lhs, lhs, rhs);
3573 #ifdef KMP_GOMP_COMPAT 3574 if (__kmp_atomic_mode == 2) {
3575 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3578 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3581 void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3582 void (*f)(
void *,
void *,
void *)) {
3583 KMP_DEBUG_ASSERT(__kmp_init_serial);
3585 #ifdef KMP_GOMP_COMPAT 3586 if (__kmp_atomic_mode == 2) {
3587 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3590 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3592 (*f)(lhs, lhs, rhs);
3594 #ifdef KMP_GOMP_COMPAT 3595 if (__kmp_atomic_mode == 2) {
3596 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3599 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3602 void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3603 void (*f)(
void *,
void *,
void *)) {
3604 KMP_DEBUG_ASSERT(__kmp_init_serial);
3606 #ifdef KMP_GOMP_COMPAT 3607 if (__kmp_atomic_mode == 2) {
3608 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3611 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3613 (*f)(lhs, lhs, rhs);
3615 #ifdef KMP_GOMP_COMPAT 3616 if (__kmp_atomic_mode == 2) {
3617 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3620 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3623 void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3624 void (*f)(
void *,
void *,
void *)) {
3625 KMP_DEBUG_ASSERT(__kmp_init_serial);
3627 #ifdef KMP_GOMP_COMPAT 3628 if (__kmp_atomic_mode == 2) {
3629 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3632 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3634 (*f)(lhs, lhs, rhs);
3636 #ifdef KMP_GOMP_COMPAT 3637 if (__kmp_atomic_mode == 2) {
3638 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3641 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3647 void __kmpc_atomic_start(
void) {
3648 int gtid = __kmp_entry_gtid();
3649 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3650 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3653 void __kmpc_atomic_end(
void) {
3654 int gtid = __kmp_get_gtid();
3655 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3656 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);