Merge branch 'chriskohlhoff:master' into zig-pkg

asio/include/asio/async_result.hpp

@@ -817,21 +817,21 @@ struct is_async_operation :
 template <typename T, typename... Args>
 ASIO_CONCEPT async_operation = is_async_operation<T, Args...>::value;
 
-#define ASIO_ASYNC_OPERATION(t) \
-  ::asio::async_operation<t>
-#define ASIO_ASYNC_OPERATION1(t, a0) \
-  ::asio::async_operation<t, a0>
-#define ASIO_ASYNC_OPERATION2(t, a0, a1) \
-  ::asio::async_operation<t, a0, a1>
-#define ASIO_ASYNC_OPERATION3(t, a0, a1, a2) \
-  ::asio::async_operation<t, a0, a1, a2>
+#define ASIO_ASYNC_OPERATION \
+  ::asio::async_operation
+#define ASIO_ASYNC_OPERATION1(a0) \
+  ::asio::async_operation<a0>
+#define ASIO_ASYNC_OPERATION2(a0, a1) \
+  ::asio::async_operation<a0, a1>
+#define ASIO_ASYNC_OPERATION3(a0, a1, a2) \
+  ::asio::async_operation<a0, a1, a2>
 
 #else // defined(ASIO_HAS_CONCEPTS)
 
-#define ASIO_ASYNC_OPERATION(t) typename
-#define ASIO_ASYNC_OPERATION1(t, a0) typename
-#define ASIO_ASYNC_OPERATION2(t, a0, a1) typename
-#define ASIO_ASYNC_OPERATION3(t, a0, a1, a2) typename
+#define ASIO_ASYNC_OPERATION typename
+#define ASIO_ASYNC_OPERATION1(a0) typename
+#define ASIO_ASYNC_OPERATION2(a0, a1) typename
+#define ASIO_ASYNC_OPERATION3(a0, a1, a2) typename
 
 #endif // defined(ASIO_HAS_CONCEPTS)
 

asio/include/asio/bind_allocator.hpp

@@ -345,14 +345,21 @@ class allocator_binder
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args)
+  result_of_t<T(Args...)> operator()(Args&&... args) &
   {
     return target_(static_cast<Args&&>(args)...);
   }
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args) const
+  result_of_t<T(Args...)> operator()(Args&&... args) &&
+  {
+    return static_cast<T&&>(target_)(static_cast<Args&&>(args)...);
+  }
+
+  /// Forwarding function call operator.
+  template <typename... Args>
+  result_of_t<T(Args...)> operator()(Args&&... args) const&
   {
     return target_(static_cast<Args&&>(args)...);
   }

asio/include/asio/bind_cancellation_slot.hpp

@@ -351,14 +351,21 @@ class cancellation_slot_binder
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args)
+  result_of_t<T(Args...)> operator()(Args&&... args) &
   {
     return target_(static_cast<Args&&>(args)...);
   }
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args) const
+  result_of_t<T(Args...)> operator()(Args&&... args) &&
+  {
+    return static_cast<T&&>(target_)(static_cast<Args&&>(args)...);
+  }
+
+  /// Forwarding function call operator.
+  template <typename... Args>
+  result_of_t<T(Args...)> operator()(Args&&... args) const&
   {
     return target_(static_cast<Args&&>(args)...);
   }

asio/include/asio/bind_executor.hpp

@@ -378,14 +378,21 @@ class executor_binder
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args)
+  result_of_t<T(Args...)> operator()(Args&&... args) &
   {
     return this->target_(static_cast<Args&&>(args)...);
   }
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args) const
+  result_of_t<T(Args...)> operator()(Args&&... args) &&
+  {
+    return static_cast<T&&>(this->target_)(static_cast<Args&&>(args)...);
+  }
+
+  /// Forwarding function call operator.
+  template <typename... Args>
+  result_of_t<T(Args...)> operator()(Args&&... args) const&
   {
     return this->target_(static_cast<Args&&>(args)...);
   }

asio/include/asio/bind_immediate_executor.hpp

@@ -350,14 +350,21 @@ class immediate_executor_binder
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args)
+  result_of_t<T(Args...)> operator()(Args&&... args) &
   {
     return target_(static_cast<Args&&>(args)...);
   }
 
   /// Forwarding function call operator.
   template <typename... Args>
-  result_of_t<T(Args...)> operator()(Args&&... args) const
+  result_of_t<T(Args...)> operator()(Args&&... args) &&
+  {
+    return static_cast<T&&>(target_)(static_cast<Args&&>(args)...);
+  }
+
+  /// Forwarding function call operator.
+  template <typename... Args>
+  result_of_t<T(Args...)> operator()(Args&&... args) const&
   {
     return target_(static_cast<Args&&>(args)...);
   }

asio/include/asio/co_composed.hpp

@@ -845,15 +845,15 @@ class co_composed_promise
     throw;
   }
 
-  template <async_operation Op>
-  auto await_transform(Op&& op
+  template <ASIO_ASYNC_OPERATION Op>
+  auto await_transform(Op&& op,
 #if defined(ASIO_ENABLE_HANDLER_TRACKING)
 # if defined(ASIO_HAS_SOURCE_LOCATION)
-      , asio::detail::source_location location
-        = asio::detail::source_location::current()
+      asio::detail::source_location location
+        = asio::detail::source_location::current(),
 # endif // defined(ASIO_HAS_SOURCE_LOCATION)
 #endif // defined(ASIO_ENABLE_HANDLER_TRACKING)
-    )
+      constraint_t<is_async_operation<Op>::value> = 0)
   {
     class [[nodiscard]] awaitable
     {

asio/include/asio/config.hpp

@@ -124,7 +124,7 @@ class config_from_concurrency_hint : public execution_context::service_maker
  * @par Example
  * @code asio::io_context my_io_context{
  *     asio::config_from_string{
- *       "scheduler.concurrency_hint=10\n
+ *       "scheduler.concurrency_hint=10\n"
  *       "scheduler.locking=1"}}; @endcode
  */
 class config_from_string : public execution_context::service_maker

asio/include/asio/detail/conditionally_enabled_mutex.hpp

@@ -70,6 +70,14 @@ class conditionally_enabled_mutex
     {
       if (mutex_.enabled_ && !locked_)
       {
+        for (int n = mutex_.spin_count_; n != 0; n -= (n > 0) ? 1 : 0)
+        {
+          if (mutex_.mutex_.try_lock())
+          {
+            locked_ = true;
+            return;
+          }
+        }
         mutex_.mutex_.lock();
         locked_ = true;
       }
@@ -104,8 +112,9 @@ class conditionally_enabled_mutex
   };
 
   // Constructor.
-  explicit conditionally_enabled_mutex(bool enabled)
-    : enabled_(enabled)
+  explicit conditionally_enabled_mutex(bool enabled, int spin_count = 0)
+    : spin_count_(spin_count),
+      enabled_(enabled)
   {
   }
 
@@ -120,11 +129,22 @@ class conditionally_enabled_mutex
     return enabled_;
   }
 
+  // Get the spin count.
+  int spin_count() const
+  {
+    return spin_count_;
+  }
+
   // Lock the mutex.
   void lock()
   {
     if (enabled_)
+    {
+      for (int n = spin_count_; n != 0; n -= (n > 0) ? 1 : 0)
+        if (mutex_.try_lock())
+          return;
       mutex_.lock();
+    }
   }
 
   // Unlock the mutex.
@@ -138,6 +158,7 @@ class conditionally_enabled_mutex
   friend class scoped_lock;
   friend class conditionally_enabled_event;
   asio::detail::mutex mutex_;
+  const int spin_count_;
   const bool enabled_;
 };
 

asio/include/asio/detail/epoll_reactor.hpp

@@ -71,7 +71,7 @@ class epoll_reactor
     bool try_speculative_[max_ops];
     bool shutdown_;
 
-    ASIO_DECL descriptor_state(bool locking);
+    ASIO_DECL descriptor_state(bool locking, int spin_count);
     void set_ready_events(uint32_t events) { task_result_ = events; }
     void add_ready_events(uint32_t events) { task_result_ |= events; }
     ASIO_DECL operation* perform_io(uint32_t events);
@@ -272,6 +272,9 @@ class epoll_reactor
   // Whether I/O locking is enabled.
   const bool io_locking_;
 
+  // How any times to spin waiting for the I/O mutex.
+  const int io_locking_spin_count_;
+
   // Mutex to protect access to the registered descriptors.
   mutex registered_descriptors_mutex_;
 

asio/include/asio/detail/impl/epoll_reactor.ipp

@@ -39,13 +39,19 @@ namespace detail {
 epoll_reactor::epoll_reactor(asio::execution_context& ctx)
   : execution_context_service_base<epoll_reactor>(ctx),
     scheduler_(use_service<scheduler>(ctx)),
-    mutex_(config(ctx).get("reactor", "registration_locking", true)),
+    mutex_(config(ctx).get("reactor", "registration_locking", true),
+        config(ctx).get("reactor", "registration_locking_spin_count", 0)),
     interrupter_(),
     epoll_fd_(do_epoll_create()),
     timer_fd_(do_timerfd_create()),
     shutdown_(false),
     io_locking_(config(ctx).get("reactor", "io_locking", true)),
-    registered_descriptors_mutex_(mutex_.enabled())
+    io_locking_spin_count_(
+        config(ctx).get("reactor", "io_locking_spin_count", 0)),
+    registered_descriptors_mutex_(mutex_.enabled(), mutex_.spin_count()),
+    registered_descriptors_(
+        config(ctx).get("reactor", "preallocated_io_objects", 0U),
+        io_locking_, io_locking_spin_count_)
 {
   // Add the interrupter's descriptor to epoll.
   epoll_event ev = { 0, { 0 } };
@@ -131,14 +137,18 @@ void epoll_reactor::notify_fork(
     for (descriptor_state* state = registered_descriptors_.first();
         state != 0; state = state->next_)
     {
-      ev.events = state->registered_events_;
-      ev.data.ptr = state;
-      int result = epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, state->descriptor_, &ev);
-      if (result != 0)
+      if (state->registered_events_ != 0)
       {
-        asio::error_code ec(errno,
-            asio::error::get_system_category());
-        asio::detail::throw_error(ec, "epoll re-registration");
+        ev.events = state->registered_events_;
+        ev.data.ptr = state;
+        int result = epoll_ctl(epoll_fd_,
+            EPOLL_CTL_ADD, state->descriptor_, &ev);
+        if (result != 0)
+        {
+          asio::error_code ec(errno,
+              asio::error::get_system_category());
+          asio::detail::throw_error(ec, "epoll re-registration");
+        }
       }
     }
   }
@@ -664,7 +674,7 @@ int epoll_reactor::do_timerfd_create()
 epoll_reactor::descriptor_state* epoll_reactor::allocate_descriptor_state()
 {
   mutex::scoped_lock descriptors_lock(registered_descriptors_mutex_);
-  return registered_descriptors_.alloc(io_locking_);
+  return registered_descriptors_.alloc(io_locking_, io_locking_spin_count_);
 }
 
 void epoll_reactor::free_descriptor_state(epoll_reactor::descriptor_state* s)
@@ -756,9 +766,9 @@ struct epoll_reactor::perform_io_cleanup_on_block_exit
   operation* first_op_;
 };
 
-epoll_reactor::descriptor_state::descriptor_state(bool locking)
+epoll_reactor::descriptor_state::descriptor_state(bool locking, int spin_count)
   : operation(&epoll_reactor::descriptor_state::do_complete),
-    mutex_(locking)
+    mutex_(locking, spin_count)
 {
 }
 

asio/include/asio/detail/impl/io_uring_service.ipp

@@ -35,15 +35,21 @@ namespace detail {
 io_uring_service::io_uring_service(asio::execution_context& ctx)
   : execution_context_service_base<io_uring_service>(ctx),
     scheduler_(use_service<scheduler>(ctx)),
-    mutex_(config(ctx).get("reactor", "registration_locking", true)),
+    mutex_(config(ctx).get("reactor", "registration_locking", true),
+        config(ctx).get("reactor", "registration_locking_spin_count", 0)),
     outstanding_work_(0),
     submit_sqes_op_(this),
     pending_sqes_(0),
     pending_submit_sqes_op_(false),
     shutdown_(false),
     io_locking_(config(ctx).get("reactor", "io_locking", true)),
+    io_locking_spin_count_(
+        config(ctx).get("reactor", "io_locking_spin_count", 0)),
     timeout_(),
     registration_mutex_(mutex_.enabled()),
+    registered_io_objects_(
+        config(ctx).get("reactor", "preallocated_io_objects", 0U),
+        io_locking_, io_locking_spin_count_),
     reactor_(use_service<reactor>(ctx)),
     reactor_data_(),
     event_fd_(-1)
@@ -612,7 +618,7 @@ void io_uring_service::register_with_reactor()
 io_uring_service::io_object* io_uring_service::allocate_io_object()
 {
   mutex::scoped_lock registration_lock(registration_mutex_);
-  return registered_io_objects_.alloc(io_locking_);
+  return registered_io_objects_.alloc(io_locking_, io_locking_spin_count_);
 }
 
 void io_uring_service::free_io_object(io_uring_service::io_object* io_obj)
@@ -897,8 +903,8 @@ void io_uring_service::io_queue::do_complete(void* owner, operation* base,
   }
 }
 
-io_uring_service::io_object::io_object(bool locking)
-  : mutex_(locking)
+io_uring_service::io_object::io_object(bool locking, int spin_count)
+  : mutex_(locking, spin_count)
 {
 }
 

asio/include/asio/detail/impl/kqueue_reactor.ipp

@@ -47,12 +47,18 @@ namespace detail {
 kqueue_reactor::kqueue_reactor(asio::execution_context& ctx)
   : execution_context_service_base<kqueue_reactor>(ctx),
     scheduler_(use_service<scheduler>(ctx)),
-    mutex_(config(ctx).get("reactor", "registration_locking", true)),
+    mutex_(config(ctx).get("reactor", "registration_locking", true),
+        config(ctx).get("reactor", "registration_locking_spin_count", 0)),
     kqueue_fd_(do_kqueue_create()),
     interrupter_(),
     shutdown_(false),
     io_locking_(config(ctx).get("reactor", "io_locking", true)),
-    registered_descriptors_mutex_(mutex_.enabled())
+    io_locking_spin_count_(
+        config(ctx).get("reactor", "io_locking_spin_count", 0)),
+    registered_descriptors_mutex_(mutex_.enabled()),
+    registered_descriptors_(
+        config(ctx).get("reactor", "preallocated_io_objects", 0U),
+        io_locking_, io_locking_spin_count_)
 {
   struct kevent events[1];
   ASIO_KQUEUE_EV_SET(&events[0], interrupter_.read_descriptor(),
@@ -563,7 +569,7 @@ int kqueue_reactor::do_kqueue_create()
 kqueue_reactor::descriptor_state* kqueue_reactor::allocate_descriptor_state()
 {
   mutex::scoped_lock descriptors_lock(registered_descriptors_mutex_);
-  return registered_descriptors_.alloc(io_locking_);
+  return registered_descriptors_.alloc(io_locking_, io_locking_spin_count_);
 }
 
 void kqueue_reactor::free_descriptor_state(kqueue_reactor::descriptor_state* s)