Index: libcxx/trunk/include/__config =================================================================== --- libcxx/trunk/include/__config +++ libcxx/trunk/include/__config @@ -38,6 +38,10 @@ # define _LIBCPP_ABI_VERSION 1 #endif +#ifndef __STDC_HOSTED__ +# define _LIBCPP_FREESTANDING +#endif + #ifndef _LIBCPP_STD_VER # if __cplusplus <= 201103L # define _LIBCPP_STD_VER 11 @@ -1220,9 +1224,18 @@ # define _LIBCPP_HAS_GCC_ATOMIC_IMP #endif -#if (!defined(_LIBCPP_HAS_C_ATOMIC_IMP) && !defined(_LIBCPP_HAS_GCC_ATOMIC_IMP)) \ +#if (!defined(_LIBCPP_HAS_C_ATOMIC_IMP) && \ + !defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) && \ + !defined(_LIBCPP_HAS_EXTERNAL_ATOMIC_IMP)) \ || defined(_LIBCPP_HAS_NO_THREADS) -#define _LIBCPP_HAS_NO_ATOMIC_HEADER +# define _LIBCPP_HAS_NO_ATOMIC_HEADER +#else +# ifndef _LIBCPP_ATOMIC_FLAG_TYPE +# define _LIBCPP_ATOMIC_FLAG_TYPE bool +# endif +# ifdef _LIBCPP_FREESTANDING +# define _LIBCPP_ATOMIC_ONLY_USE_BUILTINS +# endif #endif #ifndef _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK Index: libcxx/trunk/include/atomic =================================================================== --- libcxx/trunk/include/atomic +++ libcxx/trunk/include/atomic @@ -550,13 +550,13 @@ #endif #ifdef _LIBCPP_HAS_NO_THREADS -#error is not supported on this single threaded system +# error is not supported on this single threaded system #endif -#if !defined(_LIBCPP_HAS_C_ATOMIC_IMP) && !defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) -#error is not implemented +#ifdef _LIBCPP_HAS_NO_ATOMIC_HEADER +# error is not implemented #endif #ifdef kill_dependency -#error C++ standard library is incompatible with +# error C++ standard library is incompatible with #endif #define _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) \ @@ -583,43 +583,46 @@ memory_order_release, memory_order_acq_rel, memory_order_seq_cst } memory_order; -#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) -namespace __gcc_atomic { -template -struct __gcc_atomic_t { +#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || \ + defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS) + +// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because +// the default operator= in an object is not volatile, a byte-by-byte copy +// is required. +template _LIBCPP_INLINE_VISIBILITY +typename enable_if::value>::type +__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) { + __a_value = __val; +} +template _LIBCPP_INLINE_VISIBILITY +typename enable_if::value>::type +__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) { + volatile char* __to = reinterpret_cast(&__a_value); + volatile char* __end = __to + sizeof(_Tp); + volatile const char* __from = reinterpret_cast(&__val); + while (__to != __end) + *__to++ = *__from++; +} -#if _GNUC_VER >= 501 - static_assert(is_trivially_copyable<_Tp>::value, - "std::atomic requires that 'Tp' be a trivially copyable type"); #endif +#if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) + +template +struct __cxx_atomic_base_impl { + _LIBCPP_INLINE_VISIBILITY #ifndef _LIBCPP_CXX03_LANG - __gcc_atomic_t() _NOEXCEPT = default; + __cxx_atomic_base_impl() _NOEXCEPT = default; #else - __gcc_atomic_t() _NOEXCEPT : __a_value() {} + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} #endif // _LIBCPP_CXX03_LANG - _LIBCPP_CONSTEXPR explicit __gcc_atomic_t(_Tp value) _NOEXCEPT + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {} _Tp __a_value; }; -#define _Atomic(x) __gcc_atomic::__gcc_atomic_t - -template _Tp __create(); - -template -typename enable_if__a_value = __create<_Td>()), char>::type - __test_atomic_assignable(int); -template -__two __test_atomic_assignable(...); - -template -struct __can_assign { - static const bool value = - sizeof(__test_atomic_assignable<_Tp, _Td>(1)) == sizeof(char); -}; -static inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { +_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_order(memory_order __order) { // Avoid switch statement to make this a constexpr. return __order == memory_order_relaxed ? __ATOMIC_RELAXED: (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: @@ -629,7 +632,7 @@ __ATOMIC_CONSUME)))); } -static inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { +_LIBCPP_INLINE_VISIBILITY inline _LIBCPP_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { // Avoid switch statement to make this a constexpr. return __order == memory_order_relaxed ? __ATOMIC_RELAXED: (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: @@ -639,133 +642,125 @@ __ATOMIC_CONSUME)))); } -} // namespace __gcc_atomic - -template -static inline -typename enable_if< - __gcc_atomic::__can_assign::value>::type -__c11_atomic_init(volatile _Atomic(_Tp)* __a, _Tp __val) { - __a->__a_value = __val; -} - template -static inline -typename enable_if< - !__gcc_atomic::__can_assign::value && - __gcc_atomic::__can_assign< _Atomic(_Tp)*, _Tp>::value>::type -__c11_atomic_init(volatile _Atomic(_Tp)* __a, _Tp __val) { - // [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because - // the default operator= in an object is not volatile, a byte-by-byte copy - // is required. - volatile char* to = reinterpret_cast(&__a->__a_value); - volatile char* end = to + sizeof(_Tp); - char* from = reinterpret_cast(&__val); - while (to != end) { - *to++ = *from++; - } +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { + __cxx_atomic_assign_volatile(__a->__a_value, __val); } template -static inline void __c11_atomic_init(_Atomic(_Tp)* __a, _Tp __val) { +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { __a->__a_value = __val; } -static inline void __c11_atomic_thread_fence(memory_order __order) { - __atomic_thread_fence(__gcc_atomic::__to_gcc_order(__order)); +_LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_thread_fence(memory_order __order) { + __atomic_thread_fence(__to_gcc_order(__order)); } -static inline void __c11_atomic_signal_fence(memory_order __order) { - __atomic_signal_fence(__gcc_atomic::__to_gcc_order(__order)); +_LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_signal_fence(memory_order __order) { + __atomic_signal_fence(__to_gcc_order(__order)); } template -static inline void __c11_atomic_store(volatile _Atomic(_Tp)* __a, _Tp __val, - memory_order __order) { - return __atomic_store(&__a->__a_value, &__val, - __gcc_atomic::__to_gcc_order(__order)); +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, + memory_order __order) { + __atomic_store(&__a->__a_value, &__val, + __to_gcc_order(__order)); } template -static inline void __c11_atomic_store(_Atomic(_Tp)* __a, _Tp __val, - memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, + memory_order __order) { __atomic_store(&__a->__a_value, &__val, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_load(const volatile _Atomic(_Tp)* __a, - memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, + memory_order __order) { _Tp __ret; __atomic_load(&__a->__a_value, &__ret, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); return __ret; } template -static inline _Tp __c11_atomic_load(const _Atomic(_Tp)* __a, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { _Tp __ret; __atomic_load(&__a->__a_value, &__ret, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); return __ret; } template -static inline _Tp __c11_atomic_exchange(volatile _Atomic(_Tp)* __a, - _Tp __value, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp __value, memory_order __order) { _Tp __ret; __atomic_exchange(&__a->__a_value, &__value, &__ret, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); return __ret; } template -static inline _Tp __c11_atomic_exchange(_Atomic(_Tp)* __a, _Tp __value, - memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, + memory_order __order) { _Tp __ret; __atomic_exchange(&__a->__a_value, &__value, &__ret, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); return __ret; } template -static inline bool __c11_atomic_compare_exchange_strong( - volatile _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong( + volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, false, - __gcc_atomic::__to_gcc_order(__success), - __gcc_atomic::__to_gcc_failure_order(__failure)); + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); } template -static inline bool __c11_atomic_compare_exchange_strong( - _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, memory_order __success, +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, false, - __gcc_atomic::__to_gcc_order(__success), - __gcc_atomic::__to_gcc_failure_order(__failure)); + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); } template -static inline bool __c11_atomic_compare_exchange_weak( - volatile _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak( + volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, true, - __gcc_atomic::__to_gcc_order(__success), - __gcc_atomic::__to_gcc_failure_order(__failure)); + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); } template -static inline bool __c11_atomic_compare_exchange_weak( - _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, memory_order __success, +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak( + __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, true, - __gcc_atomic::__to_gcc_order(__success), - __gcc_atomic::__to_gcc_failure_order(__failure)); + __to_gcc_order(__success), + __to_gcc_failure_order(__failure)); } template @@ -782,80 +777,264 @@ struct __skip_amt<_Tp[n]> { }; template -static inline _Tp __c11_atomic_fetch_add(volatile _Atomic(_Tp)* __a, - _Td __delta, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, + _Td __delta, memory_order __order) { return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_add(_Atomic(_Tp)* __a, _Td __delta, - memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, + memory_order __order) { return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_sub(volatile _Atomic(_Tp)* __a, - _Td __delta, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, + _Td __delta, memory_order __order) { return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_sub(_Atomic(_Tp)* __a, _Td __delta, - memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, + memory_order __order) { return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_and(volatile _Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_and(&__a->__a_value, __pattern, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_and(_Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_and(&__a->__a_value, __pattern, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_or(volatile _Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_or(&__a->__a_value, __pattern, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_or(_Atomic(_Tp)* __a, _Tp __pattern, - memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, + memory_order __order) { return __atomic_fetch_or(&__a->__a_value, __pattern, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_xor(volatile _Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_xor(&__a->__a_value, __pattern, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_xor(_Atomic(_Tp)* __a, _Tp __pattern, - memory_order __order) { +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, + memory_order __order) { return __atomic_fetch_xor(&__a->__a_value, __pattern, - __gcc_atomic::__to_gcc_order(__order)); + __to_gcc_order(__order)); +} + +#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) + +#elif defined(_LIBCPP_HAS_C_ATOMIC_IMP) + +template +struct __cxx_atomic_base_impl { + + _LIBCPP_INLINE_VISIBILITY +#ifndef _LIBCPP_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +#else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} +#endif // _LIBCPP_CXX03_LANG + _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT + : __a_value(value) {} + _Atomic(_Tp) __a_value; +}; + +#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) + +_LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_thread_fence(int __order) { + __c11_atomic_thread_fence(__order); +} + +_LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_signal_fence(int __order) { + __c11_atomic_signal_fence(__order); +} + +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) { + __c11_atomic_init(&__a->__a_value, __val); +} +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) { + __c11_atomic_init(&__a->__a_value, __val); +} + +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, int __order) { + __c11_atomic_store(&__a->__a_value, __val, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, int __order) { + __c11_atomic_store(&__a->__a_value, __val, __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, int __order) { + using __ptr_type = typename remove_const__a_value)>::type*; + return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, int __order) { + using __ptr_type = typename remove_const__a_value)>::type*; + return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, int __order) { + return __c11_atomic_exchange(&__a->__a_value, __value, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, int __order) { + return __c11_atomic_exchange(&__a->__a_value, __value, __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, int __success, int __failure) { + return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, __success, __failure); +} +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, int __success, int __failure) { + return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, __success, __failure); +} + +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, int __success, int __failure) { + return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, __success, __failure); +} +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, int __success, int __failure) { + return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, __success, __failure); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, int __order) { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, int __order) { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, int __order) { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, int __order) { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, int __order) { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, int __order) { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, int __order) { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, int __order) { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, int __order) { + return __c11_atomic_fetch_and(&__a->__a_value, __pattern, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, int __order) { + return __c11_atomic_fetch_and(&__a->__a_value, __pattern, __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, int __order) { + return __c11_atomic_fetch_or(&__a->__a_value, __pattern, __order); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, int __order) { + return __c11_atomic_fetch_or(&__a->__a_value, __pattern, __order); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, int __order) { + return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, __order); } -#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, int __order) { + return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, __order); +} + +#endif // _LIBCPP_HAS_GCC_ATOMIC_IMP, _LIBCPP_HAS_C_ATOMIC_IMP template -inline _LIBCPP_INLINE_VISIBILITY -_Tp -kill_dependency(_Tp __y) _NOEXCEPT +_LIBCPP_INLINE_VISIBILITY +_Tp kill_dependency(_Tp __y) _NOEXCEPT { return __y; } @@ -871,7 +1050,7 @@ # define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE # define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE # define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE -#else +#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE) # define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE # define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE # define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE @@ -884,12 +1063,352 @@ # define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE #endif +#ifdef _LIBCPP_ATOMIC_ONLY_USE_BUILTINS + +template +struct __cxx_atomic_lock_impl { + + _LIBCPP_INLINE_VISIBILITY + __cxx_atomic_lock_impl() _NOEXCEPT + : __a_value(), __a_lock(0) {} + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit + __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT + : __a_value(value), __a_lock(0) {} + + _Tp __a_value; + mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock; + + _LIBCPP_INLINE_VISIBILITY void __lock() const volatile { + while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) + /*spin*/; + } + _LIBCPP_INLINE_VISIBILITY void __lock() const { + while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(true), memory_order_acquire)) + /*spin*/; + } + _LIBCPP_INLINE_VISIBILITY void __unlock() const volatile { + __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); + } + _LIBCPP_INLINE_VISIBILITY void __unlock() const { + __cxx_atomic_store(&__a_lock, _LIBCPP_ATOMIC_FLAG_TYPE(false), memory_order_release); + } + _LIBCPP_INLINE_VISIBILITY _Tp __read() const volatile { + __lock(); + _Tp __old; + __cxx_atomic_assign_volatile(__old, __a_value); + __unlock(); + return __old; + } + _LIBCPP_INLINE_VISIBILITY _Tp __read() const { + __lock(); + _Tp __old = __a_value; + __unlock(); + return __old; + } +}; + +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { + __cxx_atomic_assign_volatile(__a->__a_value, __val); +} +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val) { + __a->__a_value = __val; +} + +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { + __a->__lock(); + __cxx_atomic_assign_volatile(__a->__a_value, __val); + __a->__unlock(); +} +template +_LIBCPP_INLINE_VISIBILITY +void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) { + __a->__lock(); + __a->__a_value = __val; + __a->__unlock(); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { + return __a->__read(); +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) { + return __a->__read(); +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { + __a->__lock(); + _Tp __old; + __cxx_atomic_assign_volatile(__old, __a->__a_value); + __cxx_atomic_assign_volatile(__a->__a_value, __value); + __a->__unlock(); + return __old; +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) { + __a->__lock(); + _Tp __old = __a->__a_value; + __a->__a_value = __value; + __a->__unlock(); + return __old; +} + +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>* __a, + _Tp* __expected, _Tp __value, memory_order, memory_order) { + __a->__lock(); + _Tp temp; + __cxx_atomic_assign_volatile(temp, __a->__a_value); + bool __ret = temp == *__expected; + if(__ret) + __cxx_atomic_assign_volatile(__a->__a_value, __value); + else + __cxx_atomic_assign_volatile(*__expected, __a->__a_value); + __a->__unlock(); + return __ret; +} +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a, + _Tp* __expected, _Tp __value, memory_order, memory_order) { + __a->__lock(); + bool __ret = __a->__a_value == *__expected; + if(__ret) + __a->__a_value = __value; + else + *__expected = __a->__a_value; + __a->__unlock(); + return __ret; +} + +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __a, + _Tp* __expected, _Tp __value, memory_order, memory_order) { + __a->__lock(); + _Tp temp; + __cxx_atomic_assign_volatile(temp, __a->__a_value); + bool __ret = temp == *__expected; + if(__ret) + __cxx_atomic_assign_volatile(__a->__a_value, __value); + else + __cxx_atomic_assign_volatile(*__expected, __a->__a_value); + __a->__unlock(); + return __ret; +} +template +_LIBCPP_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a, + _Tp* __expected, _Tp __value, memory_order, memory_order) { + __a->__lock(); + bool __ret = __a->__a_value == *__expected; + if(__ret) + __a->__a_value = __value; + else + *__expected = __a->__a_value; + __a->__unlock(); + return __ret; +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp>* __a, + _Td __delta, memory_order) { + __a->__lock(); + _Tp __old; + __cxx_atomic_assign_volatile(__old, __a->__a_value); + __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta)); + __a->__unlock(); + return __old; +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp>* __a, + _Td __delta, memory_order) { + __a->__lock(); + _Tp __old = __a->__a_value; + __a->__a_value += __delta; + __a->__unlock(); + return __old; +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*>* __a, + ptrdiff_t __delta, memory_order) { + __a->__lock(); + _Tp* __old; + __cxx_atomic_assign_volatile(__old, __a->__a_value); + __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta); + __a->__unlock(); + return __old; +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*>* __a, + ptrdiff_t __delta, memory_order) { + __a->__lock(); + _Tp* __old = __a->__a_value; + __a->__a_value += __delta; + __a->__unlock(); + return __old; +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp>* __a, + _Td __delta, memory_order) { + __a->__lock(); + _Tp __old; + __cxx_atomic_assign_volatile(__old, __a->__a_value); + __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta)); + __a->__unlock(); + return __old; +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp>* __a, + _Td __delta, memory_order) { + __a->__lock(); + _Tp __old = __a->__a_value; + __a->__a_value -= __delta; + __a->__unlock(); + return __old; +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp>* __a, + _Tp __pattern, memory_order) { + __a->__lock(); + _Tp __old; + __cxx_atomic_assign_volatile(__old, __a->__a_value); + __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern)); + __a->__unlock(); + return __old; +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp>* __a, + _Tp __pattern, memory_order) { + __a->__lock(); + _Tp __old = __a->__a_value; + __a->__a_value &= __pattern; + __a->__unlock(); + return __old; +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp>* __a, + _Tp __pattern, memory_order) { + __a->__lock(); + _Tp __old; + __cxx_atomic_assign_volatile(__old, __a->__a_value); + __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern)); + __a->__unlock(); + return __old; +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp>* __a, + _Tp __pattern, memory_order) { + __a->__lock(); + _Tp __old = __a->__a_value; + __a->__a_value |= __pattern; + __a->__unlock(); + return __old; +} + +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp>* __a, + _Tp __pattern, memory_order) { + __a->__lock(); + _Tp __old; + __cxx_atomic_assign_volatile(__old, __a->__a_value); + __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern)); + __a->__unlock(); + return __old; +} +template +_LIBCPP_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp>* __a, + _Tp __pattern, memory_order) { + __a->__lock(); + _Tp __old = __a->__a_value; + __a->__a_value ^= __pattern; + __a->__unlock(); + return __old; +} + +#ifdef __cpp_lib_atomic_is_always_lock_free + +template struct __cxx_is_always_lock_free { + enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; + +#else + +template struct __cxx_is_always_lock_free { enum { __value = false }; }; +// Implementations must match the C ATOMIC_*_LOCK_FREE macro values. +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; +template struct __cxx_is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; +template<> struct __cxx_is_always_lock_free { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; + +#endif //__cpp_lib_atomic_is_always_lock_free + +template ::__value, + __cxx_atomic_base_impl<_Tp>, + __cxx_atomic_lock_impl<_Tp> >::type> +#else +template > +#endif //_LIBCPP_ATOMIC_ONLY_USE_BUILTINS +struct __cxx_atomic_impl : public _Base { + +#if _GNUC_VER >= 501 + static_assert(is_trivially_copyable<_Tp>::value, + "std::atomic requires that 'Tp' be a trivially copyable type"); +#endif + + _LIBCPP_INLINE_VISIBILITY __cxx_atomic_impl() _NOEXCEPT _LIBCPP_DEFAULT + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR explicit __cxx_atomic_impl(_Tp value) _NOEXCEPT + : _Base(value) {} +}; + // general atomic template ::value && !is_same<_Tp, bool>::value> struct __atomic_base // false { - mutable _Atomic(_Tp) __a_; + mutable __cxx_atomic_impl<_Tp> __a_; #if defined(__cpp_lib_atomic_is_always_lock_free) static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); @@ -897,88 +1416,79 @@ _LIBCPP_INLINE_VISIBILITY bool is_lock_free() const volatile _NOEXCEPT - { -#if defined(_LIBCPP_HAS_C_ATOMIC_IMP) - return __c11_atomic_is_lock_free(sizeof(_Tp)); -#else - return __atomic_is_lock_free(sizeof(_Tp), 0); -#endif - } + {return __cxx_atomic_is_lock_free(sizeof(_Tp));} _LIBCPP_INLINE_VISIBILITY bool is_lock_free() const _NOEXCEPT {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} _LIBCPP_INLINE_VISIBILITY void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) - {__c11_atomic_store(&__a_, __d, __m);} + {__cxx_atomic_store(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) - {__c11_atomic_store(&__a_, __d, __m);} + {__cxx_atomic_store(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) - {return __c11_atomic_load(&__a_, __m);} + {return __cxx_atomic_load(&__a_, __m);} _LIBCPP_INLINE_VISIBILITY _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) - {return __c11_atomic_load(&__a_, __m);} + {return __cxx_atomic_load(&__a_, __m);} _LIBCPP_INLINE_VISIBILITY operator _Tp() const volatile _NOEXCEPT {return load();} _LIBCPP_INLINE_VISIBILITY operator _Tp() const _NOEXCEPT {return load();} _LIBCPP_INLINE_VISIBILITY _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_exchange(&__a_, __d, __m);} + {return __cxx_atomic_exchange(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_exchange(&__a_, __d, __m);} + {return __cxx_atomic_exchange(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY -#ifndef _LIBCPP_CXX03_LANG - __atomic_base() _NOEXCEPT = default; -#else - __atomic_base() _NOEXCEPT : __a_() {} -#endif // _LIBCPP_CXX03_LANG + __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT + + _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR + __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} - _LIBCPP_INLINE_VISIBILITY - _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {} #ifndef _LIBCPP_CXX03_LANG __atomic_base(const __atomic_base&) = delete; __atomic_base& operator=(const __atomic_base&) = delete; @@ -1010,34 +1520,34 @@ _LIBCPP_INLINE_VISIBILITY _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_and(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_and(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_or(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_or(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_xor(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_xor(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} @@ -1119,17 +1629,17 @@ _LIBCPP_INLINE_VISIBILITY _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} @@ -1160,7 +1670,7 @@ // atomic_is_lock_free template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT { @@ -1168,7 +1678,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT { @@ -1178,25 +1688,25 @@ // atomic_init template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY void atomic_init(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { - __c11_atomic_init(&__o->__a_, __d); + __cxx_atomic_init(&__o->__a_, __d); } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY void atomic_init(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { - __c11_atomic_init(&__o->__a_, __d); + __cxx_atomic_init(&__o->__a_, __d); } // atomic_store template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY void atomic_store(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { @@ -1204,7 +1714,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY void atomic_store(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { @@ -1214,7 +1724,7 @@ // atomic_store_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY void atomic_store_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) @@ -1223,7 +1733,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY void atomic_store_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) @@ -1234,7 +1744,7 @@ // atomic_load template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT { @@ -1242,7 +1752,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT { @@ -1252,7 +1762,7 @@ // atomic_load_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) @@ -1261,7 +1771,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) @@ -1272,7 +1782,7 @@ // atomic_exchange template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_exchange(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { @@ -1280,7 +1790,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_exchange(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { @@ -1290,7 +1800,7 @@ // atomic_exchange_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_exchange_explicit(volatile atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT { @@ -1298,7 +1808,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp atomic_exchange_explicit(atomic<_Tp>* __o, _Tp __d, memory_order __m) _NOEXCEPT { @@ -1308,7 +1818,7 @@ // atomic_compare_exchange_weak template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_weak(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT { @@ -1316,7 +1826,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_weak(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT { @@ -1326,7 +1836,7 @@ // atomic_compare_exchange_strong template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_strong(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT { @@ -1334,7 +1844,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_strong(atomic<_Tp>* __o, _Tp* __e, _Tp __d) _NOEXCEPT { @@ -1344,7 +1854,7 @@ // atomic_compare_exchange_weak_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_weak_explicit(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d, @@ -1355,7 +1865,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_weak_explicit(atomic<_Tp>* __o, _Tp* __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT @@ -1367,7 +1877,7 @@ // atomic_compare_exchange_strong_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_strong_explicit(volatile atomic<_Tp>* __o, _Tp* __e, _Tp __d, @@ -1378,7 +1888,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY bool atomic_compare_exchange_strong_explicit(atomic<_Tp>* __o, _Tp* __e, _Tp __d, @@ -1391,7 +1901,7 @@ // atomic_fetch_add template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1403,7 +1913,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1415,7 +1925,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_add(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT { @@ -1423,7 +1933,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_add(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT { @@ -1433,7 +1943,7 @@ // atomic_fetch_add_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1445,7 +1955,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1457,7 +1967,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_add_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT @@ -1466,7 +1976,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_add_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT { @@ -1476,7 +1986,7 @@ // atomic_fetch_sub template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1488,7 +1998,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1500,7 +2010,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_sub(volatile atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT { @@ -1508,7 +2018,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_sub(atomic<_Tp*>* __o, ptrdiff_t __op) _NOEXCEPT { @@ -1518,7 +2028,7 @@ // atomic_fetch_sub_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1530,7 +2040,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1542,7 +2052,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_sub_explicit(volatile atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT @@ -1551,7 +2061,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY _Tp* atomic_fetch_sub_explicit(atomic<_Tp*>* __o, ptrdiff_t __op, memory_order __m) _NOEXCEPT { @@ -1561,7 +2071,7 @@ // atomic_fetch_and template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1573,7 +2083,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1587,7 +2097,7 @@ // atomic_fetch_and_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1599,7 +2109,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1613,7 +2123,7 @@ // atomic_fetch_or template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1625,7 +2135,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1639,7 +2149,7 @@ // atomic_fetch_or_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1651,7 +2161,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1665,7 +2175,7 @@ // atomic_fetch_xor template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1677,7 +2187,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1691,7 +2201,7 @@ // atomic_fetch_xor_explicit template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1703,7 +2213,7 @@ } template -inline _LIBCPP_INLINE_VISIBILITY +_LIBCPP_INLINE_VISIBILITY typename enable_if < is_integral<_Tp>::value && !is_same<_Tp, bool>::value, @@ -1718,27 +2228,23 @@ typedef struct atomic_flag { - _Atomic(bool) __a_; + __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_; _LIBCPP_INLINE_VISIBILITY bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_exchange(&__a_, true, __m);} + {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} _LIBCPP_INLINE_VISIBILITY bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_exchange(&__a_, true, __m);} + {return __cxx_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} _LIBCPP_INLINE_VISIBILITY void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {__c11_atomic_store(&__a_, false, __m);} + {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} _LIBCPP_INLINE_VISIBILITY void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {__c11_atomic_store(&__a_, false, __m);} + {__cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} _LIBCPP_INLINE_VISIBILITY -#ifndef _LIBCPP_CXX03_LANG - atomic_flag() _NOEXCEPT = default; -#else - atomic_flag() _NOEXCEPT : __a_() {} -#endif // _LIBCPP_CXX03_LANG + atomic_flag() _NOEXCEPT _LIBCPP_DEFAULT _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR atomic_flag(bool __b) _NOEXCEPT : __a_(__b) {} // EXTENSION @@ -1817,14 +2323,14 @@ void atomic_thread_fence(memory_order __m) _NOEXCEPT { - __c11_atomic_thread_fence(__m); + __cxx_atomic_thread_fence(__m); } inline _LIBCPP_INLINE_VISIBILITY void atomic_signal_fence(memory_order __m) _NOEXCEPT { - __c11_atomic_signal_fence(__m); + __cxx_atomic_signal_fence(__m); } // Atomics for standard typedef types