merge the implementations of split and ensure_started (#1176)

* merge the implementations of `split` and `ensure_started` * work around TSAN's lack of support for `std::atomic_thread_fence` * gcc-11 doesn't support `__has_feature` * add missing `#include`, suppress dumb warnings
NVIDIA · Dec 24, 2023 · d21a430 · d21a430
1 parent 6becda8
commit d21a430
Show file tree

Hide file tree

Showing 5 changed files with 504 additions and 534 deletions.
diff --git a/include/stdexec/__detail/__config.hpp b/include/stdexec/__detail/__config.hpp
@@ -204,7 +204,10 @@
 #define STDEXEC_PRAGMA_POP() _Pragma("diagnostic pop")
 #define STDEXEC_PRAGMA_IGNORE_EDG(...) _Pragma(STDEXEC_STRINGIZE(diag_suppress __VA_ARGS__))
 #elif STDEXEC_CLANG() || STDEXEC_GCC()
-#define STDEXEC_PRAGMA_PUSH() _Pragma("GCC diagnostic push")
+#define STDEXEC_PRAGMA_PUSH() \
+  _Pragma("GCC diagnostic push") STDEXEC_PRAGMA_IGNORE_GNU("-Wpragmas") STDEXEC_PRAGMA_IGNORE_GNU( \
+    "-Wunknown-pragmas") STDEXEC_PRAGMA_IGNORE_GNU("-Wunknown-warning-option") \
+    STDEXEC_PRAGMA_IGNORE_GNU("-Wunknown-attributes") STDEXEC_PRAGMA_IGNORE_GNU("-Wattributes")
 #define STDEXEC_PRAGMA_POP() _Pragma("GCC diagnostic pop")
 #define STDEXEC_PRAGMA_IGNORE_GNU(...) \
   _Pragma(STDEXEC_STRINGIZE(GCC diagnostic ignored __VA_ARGS__))
@@ -226,6 +229,12 @@
 #define STDEXEC_HAS_BUILTIN(...) 0
 #endif
 
+#if !STDEXEC_MSVC() && defined(__has_feature)
+#define STDEXEC_HAS_FEATURE __has_feature
+#else
+#define STDEXEC_HAS_FEATURE(...) 0
+#endif
+
 #if STDEXEC_HAS_BUILTIN(__is_trivially_copyable) || STDEXEC_MSVC()
 #define STDEXEC_IS_TRIVIALLY_COPYABLE(...) __is_trivially_copyable(__VA_ARGS__)
 #else
@@ -289,6 +298,12 @@
 #define STDEXEC_TERMINATE() std::terminate()
 #endif
 
+#if STDEXEC_HAS_FEATURE(thread_sanitizer) || defined(__SANITIZE_THREAD__)
+#define STDEXEC_TSAN(...) STDEXEC_HEAD_OR_TAIL(1, __VA_ARGS__)
+#else
+#define STDEXEC_TSAN(...) STDEXEC_HEAD_OR_NULL(0, __VA_ARGS__)
+#endif
+
 // Before clang-16, clang did not like libstdc++'s ranges implementation
 #if __has_include(<ranges>) && \
   (defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 201911L) && \

diff --git a/include/stdexec/__detail/__execution_fwd.hpp b/include/stdexec/__detail/__execution_fwd.hpp
@@ -191,6 +191,24 @@ namespace stdexec {
   using __bulk::bulk_t;
   extern const bulk_t bulk;
 
+  //////////////////////////////////////////////////////////////////////////////////////////////////
+  namespace __split {
+    struct split_t;
+    struct __split_t;
+  }
+
+  using __split::split_t;
+  extern const split_t split;
+
+  //////////////////////////////////////////////////////////////////////////////////////////////////
+  namespace __ensure_started {
+    struct ensure_started_t;
+    struct __ensure_started_t;
+  }
+
+  using __ensure_started::ensure_started_t;
+  extern const ensure_started_t ensure_started;
+
   //////////////////////////////////////////////////////////////////////////////////////////////////
   namespace __on_v2 {
     struct on_t;

diff --git a/include/stdexec/__detail/__intrusive_ptr.hpp b/include/stdexec/__detail/__intrusive_ptr.hpp
@@ -22,13 +22,27 @@
 #include <memory>
 #include <new>
 
+#if STDEXEC_TSAN()
+#include <sanitizer/tsan_interface.h>
+#endif
+
 namespace stdexec {
   namespace __ptr {
     template <class _Ty>
     struct __make_intrusive_t;
 
     template <class _Ty>
-    struct __enable_intrusive_from_this;
+    class __intrusive_ptr;
+
+    template <class _Ty>
+    struct __enable_intrusive_from_this {
+      __intrusive_ptr<_Ty> __intrusive_from_this() noexcept;
+      __intrusive_ptr<const _Ty> __intrusive_from_this() const noexcept;
+     private:
+      friend _Ty;
+      void __inc_ref() noexcept;
+      void __dec_ref() noexcept;
+    };
 
     template <class _Ty>
     struct __control_block {
@@ -51,6 +65,24 @@ namespace stdexec {
       _Ty& __value() const noexcept {
         return *(_Ty*) __value_;
       }
+
+      void __inc_ref_() noexcept {
+        __refcount_.fetch_add(1, std::memory_order_relaxed);
+      }
+
+      STDEXEC_PRAGMA_PUSH()
+      STDEXEC_PRAGMA_IGNORE_GNU("-Wtsan")
+
+      void __dec_ref_() noexcept {
+        if (1u == __refcount_.fetch_sub(1, std::memory_order_release)) {
+          std::atomic_thread_fence(std::memory_order_acquire);
+          // TSan does not support std::atomic_thread_fence, so we
+          // need to use the TSan-specific __tsan_acquire instead:
+          STDEXEC_TSAN(__tsan_acquire(&__refcount_));
+          delete this;
+        }
+      }
+      STDEXEC_PRAGMA_POP()
     };
 
     template <class _Ty>
@@ -65,20 +97,21 @@ namespace stdexec {
         : __data_(__data) {
       }
 
-      void __addref_() noexcept {
+      void __inc_ref_() noexcept {
         if (__data_) {
-          __data_->__refcount_.fetch_add(1, std::memory_order_relaxed);
+          __data_->__inc_ref_();
         }
       }
 
-      void __release_() noexcept {
-        if (__data_ && 1u == __data_->__refcount_.fetch_sub(1, std::memory_order_release)) {
-          std::atomic_thread_fence(std::memory_order_acquire);
-          delete __data_;
+      void __dec_ref_() noexcept {
+        if (__data_) {
+          __data_->__dec_ref_();
         }
       }
 
      public:
+      using element_type = _Ty;
+
       __intrusive_ptr() = default;
 
       __intrusive_ptr(__intrusive_ptr&& __that) noexcept
@@ -87,7 +120,11 @@ namespace stdexec {
 
       __intrusive_ptr(const __intrusive_ptr& __that) noexcept
         : __data_(__that.__data_) {
-        __addref_();
+        __inc_ref_();
+      }
+
+      __intrusive_ptr(__enable_intrusive_from_this<_Ty>* __that) noexcept
+        : __intrusive_ptr(__that ? __that->__intrusive_from_this() : __intrusive_ptr()) {
       }
 
       __intrusive_ptr& operator=(__intrusive_ptr&& __that) noexcept {
@@ -100,8 +137,12 @@ namespace stdexec {
         return operator=(__intrusive_ptr(__that));
       }
 
+      __intrusive_ptr& operator=(__enable_intrusive_from_this<_Ty>* __that) noexcept {
+        return operator=(__that ? __that->__intrusive_from_this() : __intrusive_ptr());
+      }
+
       ~__intrusive_ptr() {
-        __release_();
+        __dec_ref_();
       }
 
       void reset() noexcept {
@@ -140,23 +181,31 @@ namespace stdexec {
     };
 
     template <class _Ty>
-    struct __enable_intrusive_from_this {
-      __intrusive_ptr<_Ty> __intrusive_from_this() noexcept {
-        static_assert(0 == offsetof(__control_block<_Ty>, __value_));
-        _Ty* __this = static_cast<_Ty*>(this);
-        __intrusive_ptr<_Ty> __p{(__control_block<_Ty>*) __this};
-        __p.__addref_();
-        return __p;
-      }
+    __intrusive_ptr<_Ty> __enable_intrusive_from_this<_Ty>::__intrusive_from_this() noexcept {
+      auto* __data = (__control_block<_Ty>*) static_cast<_Ty*>(this);
+      __data->__inc_ref_();
+      return __intrusive_ptr<_Ty>{__data};
+    }
 
-      __intrusive_ptr<const _Ty> __intrusive_from_this() const noexcept {
-        static_assert(0 == offsetof(__control_block<_Ty>, __value_));
-        const _Ty* __this = static_cast<const _Ty*>(this);
-        __intrusive_ptr<const _Ty> __p{(__control_block<_Ty>*) __this};
-        __p.__addref_();
-        return __p;
-      }
-    };
+    template <class _Ty>
+    __intrusive_ptr<const _Ty>
+      __enable_intrusive_from_this<_Ty>::__intrusive_from_this() const noexcept {
+      auto* __data = (__control_block<_Ty>*) static_cast<const _Ty*>(this);
+      __data->__inc_ref_();
+      return __intrusive_ptr<const _Ty>{__data};
+    }
+
+    template <class _Ty>
+    void __enable_intrusive_from_this<_Ty>::__inc_ref() noexcept {
+      auto* __data = (__control_block<_Ty>*) static_cast<_Ty*>(this);
+      __data->__inc_ref_();
+    }
+
+    template <class _Ty>
+    void __enable_intrusive_from_this<_Ty>::__dec_ref() noexcept {
+      auto* __data = (__control_block<_Ty>*) static_cast<_Ty*>(this);
+      __data->__dec_ref_();
+    }
 
     template <class _Ty>
     struct __make_intrusive_t {

diff --git a/include/stdexec/__detail/__utility.hpp b/include/stdexec/__detail/__utility.hpp
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2023 NVIDIA Corporation
+ *
+ * Licensed under the Apache License Version 2.0 with LLVM Exceptions
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *   https://llvm.org/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "__config.hpp"
+#include "__type_traits.hpp"
+
+#include <type_traits>
+
+namespace stdexec {
+  namespace __detail {
+    template <class _Cpcvref>
+    inline constexpr auto __forward_like =
+      []<class _Uy>(_Uy&& __uy) noexcept -> auto&& {
+        return static_cast<typename _Cpcvref::template __f<std::remove_reference_t<_Uy>>>(__uy);
+      };
+  }
+
+  template <class _Ty>
+  inline constexpr auto const &__forward_like = __detail::__forward_like<__copy_cvref_fn<_Ty&&>>;
+}