[libc++] Allows any types of size 4 and 8 to use native platform ulock_wait

libcxx/docs/ABIGuarantees.rst

@@ -205,6 +205,16 @@ This flag fixes the implementation of CityHash used for ``hash<fundamental-type>
 CityHash has the problem that it drops some bits on the floor. Fixing the implementation changes the hash of values,
 resulting in an ABI break.
 
+``_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE``
+------------------------------------------
+This flag changes the implementation of ``atomic::wait()`` and ``atomic::notify_one()/notify_all()`` to use the
+native atomic wait/notify operations on platforms that support them based on the size of the atomic type, instead
+of the type itself. This means for example that a type with ``sizeof(T) == 4`` on Linux that doesn't have padding
+bytes would be able to use the underlying platform's atomic wait primitive, which is otherwise only used for ``int32_t``.
+Since the whole program must use the same implementation for correctness, changing this is an ABI break since libc++
+supports linking against TUs that were compiled against older versions of the library.
+
+
 inline namespaces
 =================
 Inline namespaces which contain types that are observable by the user need to be kept the same, since they affect

libcxx/docs/ReleaseNotes/22.rst

@@ -82,6 +82,8 @@ Improvements and New Features
 - The ``std::{generate, generate_n}`` and ``std::ranges::generate_n`` algorithms have been optimized for segmented
   iterators, resulting in a performance improvement for ``std::deque<short>`` and
   ``std::join_view<vector<vector<short>>>`` iterators.
+- ``std::atomic::wait`` has been refactored to accept more types to use platform native wait functions directly.
+  This is guarded behind the ABI Macro ``_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE``.
 
 Deprecations and Removals
 -------------------------

libcxx/include/__atomic/atomic.h

@@ -206,6 +206,8 @@ struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
 // __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
 template <class _Tp, bool _IsIntegral>
 struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
+  using __value_type _LIBCPP_NODEBUG = _Tp;
+
   static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
     return __a.load(__order);
   }

libcxx/include/__atomic/atomic_flag.h

@@ -76,6 +76,8 @@ struct atomic_flag {
 
 template <>
 struct __atomic_waitable_traits<atomic_flag> {
+  using __value_type _LIBCPP_NODEBUG = _LIBCPP_ATOMIC_FLAG_TYPE;
+
   static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) {
     return std::__cxx_atomic_load(&__a.__a_, __order);
   }

libcxx/include/__atomic/atomic_ref.h

@@ -233,6 +233,8 @@ struct __atomic_ref_base {
 
 template <class _Tp>
 struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
+  using __value_type _LIBCPP_NODEBUG = _Tp;
+
   static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {
     return __a.load(__order);
   }

libcxx/include/__atomic/atomic_sync.h

@@ -18,7 +18,10 @@
 #include <__thread/poll_with_backoff.h>
 #include <__type_traits/conjunction.h>
 #include <__type_traits/decay.h>
+#include <__type_traits/has_unique_object_representation.h>
 #include <__type_traits/invoke.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_trivially_copyable.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <cstring>
@@ -38,6 +41,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // The below implementations look ugly to support C++03
 template <class _Tp, class = void>
 struct __atomic_waitable_traits {
+  using __value_type _LIBCPP_NODEBUG = void;
+
   template <class _AtomicWaitable>
   static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
 
@@ -58,6 +63,9 @@ struct __atomic_waitable< _Tp,
 #if _LIBCPP_STD_VER >= 20
 #  if _LIBCPP_HAS_THREADS
 
+#    if !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+// old dylib interface kept for backwards compatibility
 _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
 _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
 _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*) _NOEXCEPT;
@@ -69,6 +77,114 @@ _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
 __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
 _LIBCPP_EXPORTED_FROM_ABI void
 __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT;
+#    endif // !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+// new dylib interface
+
+// return the global contention state's current value for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
+__atomic_monitor_global(void const* __address) _NOEXCEPT;
+
+// wait on the global contention state to be changed from the given value for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_global_table(void const* __address, __cxx_contention_t __monitor_value) _NOEXCEPT;
+
+// notify one waiter waiting on the global contention state for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const*) _NOEXCEPT;
+
+// notify all waiters waiting on the global contention state for the address
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const*) _NOEXCEPT;
+
+// wait on the address directly with the native platform wait
+template <std::size_t _Size>
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
+__atomic_wait_native(void const* __address, void const* __old_value) _NOEXCEPT;
+
+// notify one waiter waiting on the address directly with the native platform wait
+template <std::size_t _Size>
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(const void*) _NOEXCEPT;
+
+// notify all waiters waiting on the address directly with the native platform wait
+template <std::size_t _Size>
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(const void*) _NOEXCEPT;
+
+#    ifdef __linux__
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4)
+#    elif defined(__APPLE__)
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY)                                                               \
+        _APPLY(4)                                                                                                      \
+        _APPLY(8)
+#    elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
+#    elif defined(_WIN32)
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
+#    else
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t))
+#    endif // __linux__
+
+// concepts defines the types are supported natively by the platform's wait
+
+#    if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
+
+_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl(size_t __size) {
+  switch (__size) {
+#      define _LIBCPP_MAKE_CASE(n)                                                                                     \
+      case n:                                                                                                          \
+        return true;
+    _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE)
+  default:
+    return false;
+#      undef _LIBCPP_MAKE_CASE
+  };
+}
+
+template <class _Tp>
+concept __has_native_atomic_wait =
+    has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> &&
+    __has_native_atomic_wait_impl(sizeof(_Tp));
+
+#    else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+template <class _Tp>
+concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>;
+
+#    endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+#    if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+template <class _AtomicWaitable, class _Poll>
+struct __atomic_wait_backoff_impl {
+  const _AtomicWaitable& __a_;
+  _Poll __poll_;
+  memory_order __order_;
+
+  using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+  using __value_type _LIBCPP_NODEBUG      = typename __waitable_traits::__value_type;
+
+  _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+    if (__elapsed > chrono::microseconds(4)) {
+      auto __contention_address = const_cast<const void*>(
+          static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_)));
+
+      if constexpr (__has_native_atomic_wait<__value_type>) {
+        auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
+        if (__poll_(__atomic_value))
+          return true;
+        std::__atomic_wait_native<sizeof(__value_type)>(__contention_address, std::addressof(__atomic_value));
+      } else {
+        __cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address);
+        auto __atomic_value              = __waitable_traits::__atomic_load(__a_, __order_);
+        if (__poll_(__atomic_value))
+          return true;
+        std::__atomic_wait_global_table(__contention_address, __monitor_val);
+      }
+    } else {
+    } // poll
+    return false;
+  }
+};
+
+#    else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
 
 template <class _AtomicWaitable, class _Poll>
 struct __atomic_wait_backoff_impl {
@@ -112,6 +228,8 @@ struct __atomic_wait_backoff_impl {
   }
 };
 
+#    endif // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
 // The semantics of this function are similar to `atomic`'s
 // `.wait(T old, std::memory_order order)`, but instead of having a hardcoded
 // predicate (is the loaded value unequal to `old`?), the predicate function is
@@ -133,6 +251,38 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memo
       /* backoff */ __backoff_fn);
 }
 
+#    if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
+  using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+  auto __contention_address =
+      const_cast<const void*>(static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a)));
+  if constexpr (__has_native_atomic_wait<__value_type>) {
+    std::__atomic_notify_one_native<sizeof(__value_type)>(__contention_address);
+  } else {
+    std::__atomic_notify_one_global_table(__contention_address);
+  }
+}
+
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
+  using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+  auto __contention_address =
+      const_cast<const void*>(static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a)));
+  if constexpr (__has_native_atomic_wait<__value_type>) {
+    std::__atomic_notify_all_native<sizeof(__value_type)>(__contention_address);
+  } else {
+    std::__atomic_notify_all_global_table(__contention_address);
+  }
+}
+
+#    else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
 template <class _AtomicWaitable>
 _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
@@ -145,6 +295,8 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
   std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
 }
 
+#    endif
+
 #  else // _LIBCPP_HAS_THREADS
 
 template <class _AtomicWaitable, class _Poll>

libcxx/include/__atomic/contention_t.h

@@ -19,11 +19,35 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__))
+// The original definition of `__cxx_contention_t` seemed a bit arbitrary.
+// When we enable the _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE ABI,
+// use definitions that are based on what the underlying platform supports
+// instead.
+#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
+
+#  ifdef __linux__
+using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
+#  elif defined(__APPLE__)
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+#  elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+#  elif defined(_AIX) && !defined(__64BIT__)
+using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
+#  elif defined(_WIN32)
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+#  else
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+#  endif // __linux__
+
+#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+#  if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__))
 using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
-#else
+#  else
 using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
-#endif // __linux__ || (_AIX && !__64BIT__)
+#  endif // __linux__ || (_AIX && !__64BIT__)
+
+#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
 
 using __cxx_atomic_contention_t _LIBCPP_NODEBUG = __cxx_atomic_impl<__cxx_contention_t>;
 

libcxx/include/__configuration/abi.h

@@ -63,6 +63,7 @@
 
 // These flags are documented in ABIGuarantees.rst
 #  define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
+#  define _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
 #  define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
 #  define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
 #  define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10

libcxx/include/__configuration/availability.h

@@ -84,6 +84,9 @@
 // in all versions of the library are available.
 #if !_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS
 
+#  define _LIBCPP_INTRODUCED_IN_LLVM_22 1
+#  define _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE /* nothing */
+
 #  define _LIBCPP_INTRODUCED_IN_LLVM_21 1
 #  define _LIBCPP_INTRODUCED_IN_LLVM_21_ATTRIBUTE /* nothing */
 
@@ -112,6 +115,11 @@
 
 // clang-format off
 
+// LLVM 22
+// TODO: Fill this in
+#  define _LIBCPP_INTRODUCED_IN_LLVM_22 0
+#  define _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE __attribute__((unavailable))
+
 // LLVM 21
 // TODO: Fill this in
 #  define _LIBCPP_INTRODUCED_IN_LLVM_21 0
@@ -242,6 +250,13 @@
 
 #endif
 
+// This controls the availability of new implementation of std::atomic's
+// wait, notify_one and notify_all. The new implementation uses
+// the native atomic wait/notify operations on platforms that support them
+// based on the size of the atomic type, instead of the type itself.
+#define _LIBCPP_AVAILABILITY_HAS_NEW_SYNC _LIBCPP_INTRODUCED_IN_LLVM_22
+#define _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE
+
 // Enable additional explicit instantiations of iostreams components. This
 // reduces the number of weak definitions generated in programs that use
 // iostreams by providing a single strong definition in the shared library.

libcxx/lib/abi/CHANGELOG.TXT

@@ -16,6 +16,20 @@ New entries should be added directly below the "Version" header.
 Version 22.0
 ------------
 
+* [libc++] Allows any types of size 4 and 8 to use native platform ulock_wait
+
+  This patch added symbols for platform wait functions with the size of the type 
+
+  All platforms
+  -------------
+  Symbol added: __ZNSt3__123__atomic_monitor_globalEPKv
+  Symbol added: __ZNSt3__126__atomic_wait_global_tableEPKvx
+  Symbol added: __ZNSt3__126__atomic_notify_one_nativeILm8EEEvPKv
+  Symbol added: __ZNSt3__132__atomic_notify_all_global_tableEPKv
+  Symbol added: __ZNSt3__120__atomic_wait_nativeILm8EEEvPKvS2_
+  Symbol added: __ZNSt3__126__atomic_notify_all_nativeILm8EEEvPKv
+  Symbol added: __ZNSt3__132__atomic_notify_one_global_tableEPKv
+
 * [libc++] Remove __time_get_storage::{__analyze,init} from the ABI
 
   These functions have never been used outside the dylib, so there is no point in exporting them.