[Boost-commit] svn:boost r74826 - in trunk/boost/asio: . detail detail/impl impl

Subject: [Boost-commit] svn:boost r74826 - in trunk/boost/asio: . detail detail/impl impl
From: chris_at_[hidden]
Date: 2011-10-08 17:58:15

Author: chris_kohlhoff
Date: 2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
New Revision: 74826
URL: http://svn.boost.org/trac/boost/changeset/74826

Log:
Various performance improvements:

* Split the task_io_service's run and poll code.

* Use thread-local operation queues in single-threaded use cases (i.e. concurrency_hint is 1) to eliminate a lock/unlock pair.

* Only fence block exit when a handler is being run directly out of the io_service.

* Prefer x86 mfence-based fenced block when available.

* Use a plain ol' long for the atomic_count when all thread support is disabled.

* Allow some epoll_reactor speculative operations to be performed without holding the lock.

* Improve locality of reference by performing an epoll_reactor's I/O operation immediately before the corresponding handler is called. This also improves scalability across CPUs when multiple threads are running the io_service.

* Pass same error_code variable through to each operation's complete() function.

* Optimise creation of and access to the io_service implementation.

Text files modified: 
   trunk/boost/asio/detail/atomic_count.hpp                |     8                                         
   trunk/boost/asio/detail/call_stack.hpp                  |    67 +++++++---                              
   trunk/boost/asio/detail/completion_handler.hpp          |     5                                         
   trunk/boost/asio/detail/descriptor_read_op.hpp          |     5                                         
   trunk/boost/asio/detail/descriptor_write_op.hpp         |     5                                         
   trunk/boost/asio/detail/epoll_reactor.hpp               |    29 ++++                                    
   trunk/boost/asio/detail/fenced_block.hpp                |     8                                         
   trunk/boost/asio/detail/gcc_arm_fenced_block.hpp        |    12 +                                       
   trunk/boost/asio/detail/gcc_hppa_fenced_block.hpp       |    12 +                                       
   trunk/boost/asio/detail/gcc_sync_fenced_block.hpp       |     4                                         
   trunk/boost/asio/detail/gcc_x86_fenced_block.hpp        |    12 +                                       
   trunk/boost/asio/detail/impl/epoll_reactor.ipp          |   214 +++++++++++++++++++++++++++-------      
   trunk/boost/asio/detail/impl/kqueue_reactor.ipp         |    32 ++++                                    
   trunk/boost/asio/detail/impl/service_registry.hpp       |    18 ++                                      
   trunk/boost/asio/detail/impl/service_registry.ipp       |     6                                         
   trunk/boost/asio/detail/impl/signal_set_service.ipp     |     3                                         
   trunk/boost/asio/detail/impl/strand_service.hpp         |     2                                         
   trunk/boost/asio/detail/impl/strand_service.ipp         |     6                                         
   trunk/boost/asio/detail/impl/task_io_service.hpp        |     5                                         
   trunk/boost/asio/detail/impl/task_io_service.ipp        |   241 ++++++++++++++++++++++++++++++--------- 
   trunk/boost/asio/detail/impl/win_iocp_io_service.hpp    |     3                                         
   trunk/boost/asio/detail/impl/win_iocp_io_service.ipp    |     6                                         
   trunk/boost/asio/detail/kqueue_reactor.hpp              |     8                                         
   trunk/boost/asio/detail/macos_fenced_block.hpp          |    12 +                                       
   trunk/boost/asio/detail/null_fenced_block.hpp           |     4                                         
   trunk/boost/asio/detail/reactive_null_buffers_op.hpp    |     5                                         
   trunk/boost/asio/detail/reactive_socket_accept_op.hpp   |     5                                         
   trunk/boost/asio/detail/reactive_socket_connect_op.hpp  |     5                                         
   trunk/boost/asio/detail/reactive_socket_recv_op.hpp     |     5                                         
   trunk/boost/asio/detail/reactive_socket_recvfrom_op.hpp |     5                                         
   trunk/boost/asio/detail/reactive_socket_recvmsg_op.hpp  |     5                                         
   trunk/boost/asio/detail/reactive_socket_send_op.hpp     |     5                                         
   trunk/boost/asio/detail/reactive_socket_sendto_op.hpp   |     5                                         
   trunk/boost/asio/detail/resolve_endpoint_op.hpp         |     5                                         
   trunk/boost/asio/detail/resolve_op.hpp                  |     5                                         
   trunk/boost/asio/detail/service_registry.hpp            |    15 +                                       
   trunk/boost/asio/detail/signal_handler.hpp              |     5                                         
   trunk/boost/asio/detail/signal_set_service.hpp          |     2                                         
   trunk/boost/asio/detail/solaris_fenced_block.hpp        |    12 +                                       
   trunk/boost/asio/detail/strand_service.hpp              |     2                                         
   trunk/boost/asio/detail/task_io_service.hpp             |    41 ++++-                                   
   trunk/boost/asio/detail/task_io_service_operation.hpp   |    13 +                                       
   trunk/boost/asio/detail/wait_handler.hpp                |     5                                         
   trunk/boost/asio/detail/win_fenced_block.hpp            |    12 +                                       
   trunk/boost/asio/detail/win_iocp_handle_read_op.hpp     |     7                                         
   trunk/boost/asio/detail/win_iocp_handle_write_op.hpp    |     4                                         
   trunk/boost/asio/detail/win_iocp_io_service.hpp         |    14 +                                       
   trunk/boost/asio/detail/win_iocp_null_buffers_op.hpp    |     7                                         
   trunk/boost/asio/detail/win_iocp_operation.hpp          |     9                                         
   trunk/boost/asio/detail/win_iocp_overlapped_op.hpp      |     4                                         
   trunk/boost/asio/detail/win_iocp_socket_accept_op.hpp   |     7                                         
   trunk/boost/asio/detail/win_iocp_socket_recv_op.hpp     |     7                                         
   trunk/boost/asio/detail/win_iocp_socket_recvfrom_op.hpp |     7                                         
   trunk/boost/asio/detail/win_iocp_socket_recvmsg_op.hpp  |     7                                         
   trunk/boost/asio/detail/win_iocp_socket_send_op.hpp     |     7                                         
   trunk/boost/asio/impl/io_service.hpp                    |    19 ++                                      
   trunk/boost/asio/impl/io_service.ipp                    |    14 +                                       
   trunk/boost/asio/io_service.hpp                         |     4                                         
   58 files changed, 736 insertions(+), 265 deletions(-)

Modified: trunk/boost/asio/detail/atomic_count.hpp
==============================================================================
--- trunk/boost/asio/detail/atomic_count.hpp	(original)
+++ trunk/boost/asio/detail/atomic_count.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -17,7 +17,9 @@
 
 #include <boost/asio/detail/config.hpp>
 
-#if defined(BOOST_ASIO_HAS_STD_ATOMIC)
+#if !defined(BOOST_HAS_THREADS) || defined(BOOST_ASIO_DISABLE_THREADS)
+// Nothing to include.
+#elif defined(BOOST_ASIO_HAS_STD_ATOMIC)
 # include <atomic>
 #else // defined(BOOST_ASIO_HAS_STD_ATOMIC)
 # include <boost/detail/atomic_count.hpp>
@@ -27,7 +29,9 @@
 namespace asio {
 namespace detail {
 
-#if defined(BOOST_ASIO_HAS_STD_ATOMIC)
+#if !defined(BOOST_HAS_THREADS) || defined(BOOST_ASIO_DISABLE_THREADS)
+typedef long atomic_count;
+#elif defined(BOOST_ASIO_HAS_STD_ATOMIC)
 typedef std::atomic<long> atomic_count;
 #else // defined(BOOST_ASIO_HAS_STD_ATOMIC)
 typedef boost::detail::atomic_count atomic_count;

Modified: trunk/boost/asio/detail/call_stack.hpp
==============================================================================
--- trunk/boost/asio/detail/call_stack.hpp	(original)
+++ trunk/boost/asio/detail/call_stack.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -27,34 +27,60 @@
 
 // Helper class to determine whether or not the current thread is inside an
 // invocation of io_service::run() for a specified io_service object.
-template <typename Owner>
+template <typename Key, typename Value = unsigned char>
 class call_stack
 {
 public:
-  // Context class automatically pushes an owner on to the stack.
+  // Context class automatically pushes the key/value pair on to the stack.
   class context
     : private noncopyable
   {
   public:
-    // Push the owner on to the stack.
-    explicit context(Owner* d)
-      : owner_(d),
-        next_(call_stack<Owner>::top_)
+    // Push the key on to the stack.
+    explicit context(Key* k)
+      : key_(k),
+        next_(call_stack<Key, Value>::top_)
     {
-      call_stack<Owner>::top_ = this;
+      value_ = reinterpret_cast<unsigned char*>(this);
+      call_stack<Key, Value>::top_ = this;
     }
 
-    // Pop the owner from the stack.
+    // Push the key/value pair on to the stack.
+    context(Key* k, Value& v)
+      : key_(k),
+        value_(&v),
+        next_(call_stack<Key, Value>::top_)
+    {
+      call_stack<Key, Value>::top_ = this;
+    }
+
+    // Pop the key/value pair from the stack.
     ~context()
     {
-      call_stack<Owner>::top_ = next_;
+      call_stack<Key, Value>::top_ = next_;
+    }
+
+    // Find the next context with the same key.
+    Value* next_by_key() const
+    {
+      context* elem = next_;
+      while (elem)
+      {
+        if (elem->key_ == key_)
+          return elem->value_;
+        elem = elem->next_;
+      }
+      return 0;
     }
 
   private:
-    friend class call_stack<Owner>;
+    friend class call_stack<Key, Value>;
+
+    // The key associated with the context.
+    Key* key_;
 
-    // The owner associated with the context.
-    Owner* owner_;
+    // The value associated with the context.
+    Value* value_;
 
     // The next element in the stack.
     context* next_;
@@ -62,17 +88,18 @@
 
   friend class context;
 
-  // Determine whether the specified owner is on the stack.
-  static bool contains(Owner* d)
+  // Determine whether the specified owner is on the stack. Returns address of
+  // key if present, 0 otherwise.
+  static Value* contains(Key* k)
   {
     context* elem = top_;
     while (elem)
     {
-      if (elem->owner_ == d)
-        return true;
+      if (elem->key_ == k)
+        return elem->value_;
       elem = elem->next_;
     }
-    return false;
+    return 0;
   }
 
 private:
@@ -80,9 +107,9 @@
   static tss_ptr<context> top_;
 };
 
-template <typename Owner>
-tss_ptr<typename call_stack<Owner>::context>
-call_stack<Owner>::top_;
+template <typename Key, typename Value>
+tss_ptr<typename call_stack<Key, Value>::context>
+call_stack<Key, Value>::top_;
 
 } // namespace detail
 } // namespace asio

Modified: trunk/boost/asio/detail/completion_handler.hpp
==============================================================================
--- trunk/boost/asio/detail/completion_handler.hpp	(original)
+++ trunk/boost/asio/detail/completion_handler.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -40,7 +40,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     completion_handler* h(static_cast<completion_handler*>(base));
@@ -61,7 +62,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN(());
       boost_asio_handler_invoke_helpers::invoke(handler, handler);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/descriptor_read_op.hpp
==============================================================================
--- trunk/boost/asio/detail/descriptor_read_op.hpp	(original)
+++ trunk/boost/asio/detail/descriptor_read_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -76,7 +76,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     descriptor_read_op* o(static_cast<descriptor_read_op*>(base));
@@ -98,7 +99,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/descriptor_write_op.hpp
==============================================================================
--- trunk/boost/asio/detail/descriptor_write_op.hpp	(original)
+++ trunk/boost/asio/detail/descriptor_write_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -76,7 +76,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     descriptor_write_op* o(static_cast<descriptor_write_op*>(base));
@@ -98,7 +99,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/epoll_reactor.hpp
==============================================================================
--- trunk/boost/asio/detail/epoll_reactor.hpp	(original)
+++ trunk/boost/asio/detail/epoll_reactor.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -21,6 +21,7 @@
 
 #include <boost/limits.hpp>
 #include <boost/asio/io_service.hpp>
+#include <boost/asio/detail/atomic_count.hpp>
 #include <boost/asio/detail/epoll_reactor_fwd.hpp>
 #include <boost/asio/detail/mutex.hpp>
 #include <boost/asio/detail/object_pool.hpp>
@@ -47,16 +48,28 @@
     connect_op = 1, except_op = 2, max_ops = 3 };
 
   // Per-descriptor queues.
-  class descriptor_state
+  class descriptor_state : operation
   {
     friend class epoll_reactor;
     friend class object_pool_access;
+
+    descriptor_state* next_;
+    descriptor_state* prev_;
+
+    bool op_queue_is_empty_[max_ops];
+
     mutex mutex_;
+    epoll_reactor* reactor_;
     int descriptor_;
     op_queue<reactor_op> op_queue_[max_ops];
     bool shutdown_;
-    descriptor_state* next_;
-    descriptor_state* prev_;
+
+    BOOST_ASIO_DECL descriptor_state();
+    void set_ready_events(uint32_t events) { task_result_ = events; }
+    BOOST_ASIO_DECL operation* perform_io(uint32_t events);
+    BOOST_ASIO_DECL static void do_complete(
+        io_service_impl* owner, operation* base,
+        const boost::system::error_code& ec, std::size_t bytes_transferred);
   };
 
   // Per-descriptor data.
@@ -160,6 +173,12 @@
   // Create the timerfd file descriptor. Does not throw.
   BOOST_ASIO_DECL static int do_timerfd_create();
 
+  // Allocate a new descriptor state object.
+  BOOST_ASIO_DECL descriptor_state* allocate_descriptor_state();
+
+  // Free an existing descriptor state object.
+  BOOST_ASIO_DECL void free_descriptor_state(descriptor_state* s);
+
   // Helper function to add a new timer queue.
   BOOST_ASIO_DECL void do_add_timer_queue(timer_queue_base& queue);
 
@@ -206,6 +225,10 @@
 
   // Keep track of all registered descriptors.
   object_pool<descriptor_state> registered_descriptors_;
+
+  // Helper class to do post-perform_io cleanup.
+  struct perform_io_cleanup_on_block_exit;
+  friend struct perform_io_cleanup_on_block_exit;
 };
 
 } // namespace detail

Modified: trunk/boost/asio/detail/fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -29,13 +29,13 @@
 # include <boost/asio/detail/gcc_arm_fenced_block.hpp>
 #elif defined(__GNUC__) && (defined(__hppa) || defined(__hppa__))
 # include <boost/asio/detail/gcc_hppa_fenced_block.hpp>
+#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+# include <boost/asio/detail/gcc_x86_fenced_block.hpp>
 #elif defined(__GNUC__) \
   && ((__GNUC__ == 4 && __GNUC_MINOR__ >= 1) || (__GNUC__ > 4)) \
   && !defined(__INTEL_COMPILER) && !defined(__ICL) \
   && !defined(__ICC) && !defined(__ECC) && !defined(__PATHSCALE__)
 # include <boost/asio/detail/gcc_sync_fenced_block.hpp>
-#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-# include <boost/asio/detail/gcc_x86_fenced_block.hpp>
 #elif defined(BOOST_WINDOWS) && !defined(UNDER_CE)
 # include <boost/asio/detail/win_fenced_block.hpp>
 #else
@@ -58,13 +58,13 @@
 typedef gcc_arm_fenced_block fenced_block;
 #elif defined(__GNUC__) && (defined(__hppa) || defined(__hppa__))
 typedef gcc_hppa_fenced_block fenced_block;
+#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+typedef gcc_x86_fenced_block fenced_block;
 #elif defined(__GNUC__) \
   && ((__GNUC__ == 4 && __GNUC_MINOR__ >= 1) || (__GNUC__ > 4)) \
   && !defined(__INTEL_COMPILER) && !defined(__ICL) \
   && !defined(__ICC) && !defined(__ECC) && !defined(__PATHSCALE__)
 typedef gcc_sync_fenced_block fenced_block;
-#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-typedef gcc_x86_fenced_block fenced_block;
 #elif defined(BOOST_WINDOWS) && !defined(UNDER_CE)
 typedef win_fenced_block fenced_block;
 #else

Modified: trunk/boost/asio/detail/gcc_arm_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/gcc_arm_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/gcc_arm_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -29,8 +29,16 @@
   : private noncopyable
 {
 public:
-  // Constructor.
-  gcc_arm_fenced_block()
+  enum half_t { half };
+  enum full_t { full };
+
+  // Constructor for a half fenced block.
+  explicit gcc_arm_fenced_block(half_t)
+  {
+  }
+
+  // Constructor for a full fenced block.
+  explicit gcc_arm_fenced_block(full_t)
   {
     barrier();
   }

Modified: trunk/boost/asio/detail/gcc_hppa_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/gcc_hppa_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/gcc_hppa_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -29,8 +29,16 @@
   : private noncopyable
 {
 public:
-  // Constructor.
-  gcc_hppa_fenced_block()
+  enum half_t { half };
+  enum full_t { full };
+
+  // Constructor for a half fenced block.
+  explicit gcc_hppa_fenced_block(half_t)
+  {
+  }
+
+  // Constructor for a full fenced block.
+  explicit gcc_hppa_fenced_block(full_t)
   {
     barrier();
   }

Modified: trunk/boost/asio/detail/gcc_sync_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/gcc_sync_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/gcc_sync_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -32,8 +32,10 @@
   : private noncopyable
 {
 public:
+  enum half_or_full_t { half, full };
+
   // Constructor.
-  gcc_sync_fenced_block()
+  explicit gcc_sync_fenced_block(half_or_full_t)
     : value_(0)
   {
     __sync_lock_test_and_set(&value_, 1);

Modified: trunk/boost/asio/detail/gcc_x86_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/gcc_x86_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/gcc_x86_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -29,8 +29,16 @@
   : private noncopyable
 {
 public:
-  // Constructor.
-  gcc_x86_fenced_block()
+  enum half_t { half };
+  enum full_t { full };
+
+  // Constructor for a half fenced block.
+  explicit gcc_x86_fenced_block(half_t)
+  {
+  }
+
+  // Constructor for a full fenced block.
+  explicit gcc_x86_fenced_block(full_t)
   {
     barrier1();
   }

Modified: trunk/boost/asio/detail/impl/epoll_reactor.ipp
==============================================================================
--- trunk/boost/asio/detail/impl/epoll_reactor.ipp	(original)
+++ trunk/boost/asio/detail/impl/epoll_reactor.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -148,13 +148,19 @@
 int epoll_reactor::register_descriptor(socket_type descriptor,
     epoll_reactor::per_descriptor_data& descriptor_data)
 {
-  mutex::scoped_lock lock(registered_descriptors_mutex_);
+  descriptor_data = allocate_descriptor_state();
 
-  descriptor_data = registered_descriptors_.alloc();
-  descriptor_data->descriptor_ = descriptor;
-  descriptor_data->shutdown_ = false;
+  {
+    mutex::scoped_lock descriptor_lock(descriptor_data->mutex_);
 
-  lock.unlock();
+    descriptor_data->reactor_ = this;
+    descriptor_data->descriptor_ = descriptor;
+    descriptor_data->shutdown_ = false;
+
+    for (int i = 0; i < max_ops; ++i)
+      descriptor_data->op_queue_is_empty_[i] =
+        descriptor_data->op_queue_[i].empty();
+  }
 
   epoll_event ev = { 0, { 0 } };
   ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLOUT | EPOLLPRI | EPOLLET;
@@ -170,14 +176,20 @@
     int op_type, socket_type descriptor,
     epoll_reactor::per_descriptor_data& descriptor_data, reactor_op* op)
 {
-  mutex::scoped_lock lock(registered_descriptors_mutex_);
+  descriptor_data = allocate_descriptor_state();
 
-  descriptor_data = registered_descriptors_.alloc();
-  descriptor_data->descriptor_ = descriptor;
-  descriptor_data->shutdown_ = false;
-  descriptor_data->op_queue_[op_type].push(op);
+  {
+    mutex::scoped_lock descriptor_lock(descriptor_data->mutex_);
 
-  lock.unlock();
+    descriptor_data->reactor_ = this;
+    descriptor_data->descriptor_ = descriptor;
+    descriptor_data->shutdown_ = false;
+    descriptor_data->op_queue_[op_type].push(op);
+
+    for (int i = 0; i < max_ops; ++i)
+      descriptor_data->op_queue_is_empty_[i] =
+        descriptor_data->op_queue_[i].empty();
+  }
 
   epoll_event ev = { 0, { 0 } };
   ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLOUT | EPOLLPRI | EPOLLET;
@@ -208,6 +220,22 @@
     return;
   }
 
+  bool perform_speculative = allow_speculative;
+  if (perform_speculative)
+  {
+    if (descriptor_data->op_queue_is_empty_[op_type]
+        && (op_type != read_op
+          || descriptor_data->op_queue_is_empty_[except_op]))
+    {
+      if (op->perform())
+      {
+        io_service_.post_immediate_completion(op);
+        return;
+      }
+      perform_speculative = false;
+    }
+  }
+
   mutex::scoped_lock descriptor_lock(descriptor_data->mutex_);
 
   if (descriptor_data->shutdown_)
@@ -216,17 +244,24 @@
     return;
   }
 
-  if (descriptor_data->op_queue_[op_type].empty())
+  for (int i = 0; i < max_ops; ++i)
+    descriptor_data->op_queue_is_empty_[i] =
+      descriptor_data->op_queue_[i].empty();
+
+  if (descriptor_data->op_queue_is_empty_[op_type])
   {
-    if (allow_speculative
-        && (op_type != read_op
-          || descriptor_data->op_queue_[except_op].empty()))
+    if (allow_speculative)
     {
-      if (op->perform())
+      if (perform_speculative
+          && (op_type != read_op
+            || descriptor_data->op_queue_is_empty_[except_op]))
       {
-        descriptor_lock.unlock();
-        io_service_.post_immediate_completion(op);
-        return;
+        if (op->perform())
+        {
+          descriptor_lock.unlock();
+          io_service_.post_immediate_completion(op);
+          return;
+        }
       }
     }
     else
@@ -240,6 +275,7 @@
   }
 
   descriptor_data->op_queue_[op_type].push(op);
+  descriptor_data->op_queue_is_empty_[op_type] = false;
   io_service_.work_started();
 }
 
@@ -274,7 +310,6 @@
     return;
 
   mutex::scoped_lock descriptor_lock(descriptor_data->mutex_);
-  mutex::scoped_lock descriptors_lock(registered_descriptors_mutex_);
 
   if (!descriptor_data->shutdown_)
   {
@@ -305,11 +340,9 @@
 
     descriptor_lock.unlock();
 
-    registered_descriptors_.free(descriptor_data);
+    free_descriptor_state(descriptor_data);
     descriptor_data = 0;
 
-    descriptors_lock.unlock();
-
     io_service_.post_deferred_completions(ops);
   }
 }
@@ -321,7 +354,6 @@
     return;
 
   mutex::scoped_lock descriptor_lock(descriptor_data->mutex_);
-  mutex::scoped_lock descriptors_lock(registered_descriptors_mutex_);
 
   if (!descriptor_data->shutdown_)
   {
@@ -337,15 +369,19 @@
 
     descriptor_lock.unlock();
 
-    registered_descriptors_.free(descriptor_data);
+    free_descriptor_state(descriptor_data);
     descriptor_data = 0;
-
-    descriptors_lock.unlock();
   }
 }
 
 void epoll_reactor::run(bool block, op_queue<operation>& ops)
 {
+  // This code relies on the fact that the task_io_service queues the reactor
+  // task behind all descriptor operations generated by this function. This
+  // means, that by the time we reach this point, any previously returned
+  // descriptor operations have already been dequeued. Therefore it is now safe
+  // for us to reuse and return them for the task_io_service to queue again.
+
   // Calculate a timeout only if timerfd is not used.
   int timeout;
   if (timer_fd_ != -1)
@@ -392,28 +428,12 @@
 #endif // defined(BOOST_ASIO_HAS_TIMERFD)
     else
     {
+      // The descriptor operation doesn't count as work in and of itself, so we
+      // don't call work_started() here. This still allows the io_service to
+      // stop if the only remaining operations are descriptor operations.
       descriptor_state* descriptor_data = static_cast<descriptor_state*>(ptr);
-      mutex::scoped_lock descriptor_lock(descriptor_data->mutex_);
-
-      // Exception operations must be processed first to ensure that any
-      // out-of-band data is read before normal data.
-      static const int flag[max_ops] = { EPOLLIN, EPOLLOUT, EPOLLPRI };
-      for (int j = max_ops - 1; j >= 0; --j)
-      {
-        if (events[i].events & (flag[j] | EPOLLERR | EPOLLHUP))
-        {
-          while (reactor_op* op = descriptor_data->op_queue_[j].front())
-          {
-            if (op->perform())
-            {
-              descriptor_data->op_queue_[j].pop();
-              ops.push(op);
-            }
-            else
-              break;
-          }
-        }
-      }
+      descriptor_data->set_ready_events(events[i].events);
+      ops.push(descriptor_data);
     }
   }
 
@@ -491,6 +511,18 @@
 #endif // defined(BOOST_ASIO_HAS_TIMERFD)
 }
 
+epoll_reactor::descriptor_state* epoll_reactor::allocate_descriptor_state()
+{
+  mutex::scoped_lock descriptors_lock(registered_descriptors_mutex_);
+  return registered_descriptors_.alloc();
+}
+
+void epoll_reactor::free_descriptor_state(epoll_reactor::descriptor_state* s)
+{
+  mutex::scoped_lock descriptors_lock(registered_descriptors_mutex_);
+  registered_descriptors_.free(s);
+}
+
 void epoll_reactor::do_add_timer_queue(timer_queue_base& queue)
 {
   mutex::scoped_lock lock(mutex_);
@@ -539,6 +571,92 @@
 }
 #endif // defined(BOOST_ASIO_HAS_TIMERFD)
 
+struct epoll_reactor::perform_io_cleanup_on_block_exit
+{
+  explicit perform_io_cleanup_on_block_exit(epoll_reactor* r)
+    : reactor_(r), first_op_(0)
+  {
+  }
+
+  ~perform_io_cleanup_on_block_exit()
+  {
+    if (first_op_)
+    {
+      // Post the remaining completed operations for invocation.
+      if (!ops_.empty())
+        reactor_->io_service_.post_deferred_completions(ops_);
+
+      // A user-initiated operation has completed, but there's no need to
+      // explicitly call work_finished() here. Instead, we'll take advantage of
+      // the fact that the task_io_service will call work_finished() once we
+      // return.
+    }
+    else
+    {
+      // No user-initiated operations have completed, so we need to compensate
+      // for the work_finished() call that the task_io_service will make once
+      // this operation returns.
+      reactor_->io_service_.work_started();
+    }
+  }
+
+  epoll_reactor* reactor_;
+  op_queue<operation> ops_;
+  operation* first_op_;
+};
+
+epoll_reactor::descriptor_state::descriptor_state()
+  : operation(&epoll_reactor::descriptor_state::do_complete)
+{
+}
+
+operation* epoll_reactor::descriptor_state::perform_io(uint32_t events)
+{
+  perform_io_cleanup_on_block_exit io_cleanup(reactor_);
+  mutex::scoped_lock descriptor_lock(mutex_);
+
+  // Exception operations must be processed first to ensure that any
+  // out-of-band data is read before normal data.
+  static const int flag[max_ops] = { EPOLLIN, EPOLLOUT, EPOLLPRI };
+  for (int j = max_ops - 1; j >= 0; --j)
+  {
+    if (events & (flag[j] | EPOLLERR | EPOLLHUP))
+    {
+      while (reactor_op* op = op_queue_[j].front())
+      {
+        if (op->perform())
+        {
+          op_queue_[j].pop();
+          io_cleanup.ops_.push(op);
+        }
+        else
+          break;
+      }
+    }
+  }
+
+  // The first operation will be returned for completion now. The others will
+  // be posted for later by the io_cleanup object's destructor.
+  io_cleanup.first_op_ = io_cleanup.ops_.front();
+  io_cleanup.ops_.pop();
+  return io_cleanup.first_op_;
+}
+
+void epoll_reactor::descriptor_state::do_complete(
+    io_service_impl* owner, operation* base,
+    const boost::system::error_code& ec, std::size_t bytes_transferred)
+{
+  if (owner)
+  {
+    descriptor_state* descriptor_data = static_cast<descriptor_state*>(base);
+    uint32_t events = static_cast<uint32_t>(bytes_transferred);
+    if (operation* op = descriptor_data->perform_io(events))
+    {
+      op->complete(*owner, ec, 0);
+    }
+  }
+}
+
 } // namespace detail
 } // namespace asio
 } // namespace boost

Modified: trunk/boost/asio/detail/impl/kqueue_reactor.ipp
==============================================================================
--- trunk/boost/asio/detail/impl/kqueue_reactor.ipp	(original)
+++ trunk/boost/asio/detail/impl/kqueue_reactor.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -132,6 +132,10 @@
   descriptor_data->descriptor_ = descriptor;
   descriptor_data->shutdown_ = false;
 
+  for (int i = 0; i < max_ops; ++i)
+    descriptor_data->op_queue_is_empty_[i] =
+      descriptor_data->op_queue_[i].empty();
+
   return 0;
 }
 
@@ -146,6 +150,10 @@
   descriptor_data->shutdown_ = false;
   descriptor_data->op_queue_[op_type].push(op);
 
+  for (int i = 0; i < max_ops; ++i)
+    descriptor_data->op_queue_is_empty_[i] =
+      descriptor_data->op_queue_[i].empty();
+
   struct kevent event;
   switch (op_type)
   {
@@ -186,6 +194,21 @@
     return;
   }
 
+  if (allow_speculative)
+  {
+    if (descriptor_data->op_queue_is_empty_[op_type]
+        && (op_type != read_op
+          || descriptor_data->op_queue_is_empty_[except_op]))
+    {
+      if (op->perform())
+      {
+        io_service_.post_immediate_completion(op);
+        return;
+      }
+      allow_speculative = false;
+    }
+  }
+
   mutex::scoped_lock descriptor_lock(descriptor_data->mutex_);
 
   if (descriptor_data->shutdown_)
@@ -194,12 +217,16 @@
     return;
   }
 
-  bool first = descriptor_data->op_queue_[op_type].empty();
+  for (int i = 0; i < max_ops; ++i)
+    descriptor_data->op_queue_is_empty_[i] =
+      descriptor_data->op_queue_[i].empty();
+
+  bool first = descriptor_data->op_queue_is_empty_[op_type];
   if (first)
   {
     if (allow_speculative)
     {
-      if (op_type != read_op || descriptor_data->op_queue_[except_op].empty())
+      if (op_type != read_op || descriptor_data->op_queue_is_empty_[except_op])
       {
         if (op->perform())
         {
@@ -212,6 +239,7 @@
   }
 
   descriptor_data->op_queue_[op_type].push(op);
+  descriptor_data->op_queue_is_empty_[op_type] = false;
   io_service_.work_started();
 
   if (first)

Modified: trunk/boost/asio/detail/impl/service_registry.hpp
==============================================================================
--- trunk/boost/asio/detail/impl/service_registry.hpp	(original)
+++ trunk/boost/asio/detail/impl/service_registry.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -21,6 +21,24 @@
 namespace asio {
 namespace detail {
 
+template <typename Service, typename Arg>
+service_registry::service_registry(
+    boost::asio::io_service& o, Service*, Arg arg)
+  : owner_(o),
+    first_service_(new Service(o, arg))
+{
+  boost::asio::io_service::service::key key;
+  init_key(key, Service::id);
+  first_service_->key_ = key;
+  first_service_->next_ = 0;
+}
+
+template <typename Service>
+Service& service_registry::first_service()
+{
+  return *static_cast<Service*>(first_service_);
+}
+
 template <typename Service>
 Service& service_registry::use_service()
 {

Modified: trunk/boost/asio/detail/impl/service_registry.ipp
==============================================================================
--- trunk/boost/asio/detail/impl/service_registry.ipp	(original)
+++ trunk/boost/asio/detail/impl/service_registry.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -26,12 +26,6 @@
 namespace asio {
 namespace detail {
 
-service_registry::service_registry(boost::asio::io_service& o)
-  : owner_(o),
-    first_service_(0)
-{
-}
-
 service_registry::~service_registry()
 {
   // Shutdown all services. This must be done in a separate loop before the

Modified: trunk/boost/asio/detail/impl/signal_set_service.ipp
==============================================================================
--- trunk/boost/asio/detail/impl/signal_set_service.ipp	(original)
+++ trunk/boost/asio/detail/impl/signal_set_service.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -98,7 +98,8 @@
   }
 
   static void do_complete(io_service_impl* /*owner*/, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     pipe_read_op* o(static_cast<pipe_read_op*>(base));
     delete o;

Modified: trunk/boost/asio/detail/impl/strand_service.hpp
==============================================================================
--- trunk/boost/asio/detail/impl/strand_service.hpp	(original)
+++ trunk/boost/asio/detail/impl/strand_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -61,7 +61,7 @@
   // If we are already in the strand then the handler can run immediately.
   if (call_stack<strand_impl>::contains(impl))
   {
-    boost::asio::detail::fenced_block b;
+    fenced_block b(fenced_block::full);
     boost_asio_handler_invoke_helpers::invoke(handler, handler);
     return;
   }

Modified: trunk/boost/asio/detail/impl/strand_service.ipp
==============================================================================
--- trunk/boost/asio/detail/impl/strand_service.ipp	(original)
+++ trunk/boost/asio/detail/impl/strand_service.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -79,7 +79,7 @@
 {
   // If we are running inside the io_service, and no other handler is queued
   // or running, then the handler can run immediately.
-  bool can_dispatch = call_stack<io_service_impl>::contains(&io_service_);
+  bool can_dispatch = io_service_.can_dispatch();
   impl->mutex_.lock();
   bool first = (++impl->count_ == 1);
   if (can_dispatch && first)
@@ -115,7 +115,7 @@
 }
 
 void strand_service::do_complete(io_service_impl* owner, operation* base,
-    boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+    const boost::system::error_code& ec, std::size_t /*bytes_transferred*/)
 {
   if (owner)
   {
@@ -134,7 +134,7 @@
     on_do_complete_exit on_exit = { owner, impl };
     (void)on_exit;
 
-    o->complete(*owner);
+    o->complete(*owner, ec, 0);
   }
 }
 

Modified: trunk/boost/asio/detail/impl/task_io_service.hpp
==============================================================================
--- trunk/boost/asio/detail/impl/task_io_service.hpp	(original)
+++ trunk/boost/asio/detail/impl/task_io_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -15,7 +15,6 @@
 # pragma once
 #endif // defined(_MSC_VER) && (_MSC_VER >= 1200)
 
-#include <boost/asio/detail/call_stack.hpp>
 #include <boost/asio/detail/completion_handler.hpp>
 #include <boost/asio/detail/fenced_block.hpp>
 #include <boost/asio/detail/handler_alloc_helpers.hpp>
@@ -30,9 +29,9 @@
 template <typename Handler>
 void task_io_service::dispatch(Handler handler)
 {
-  if (call_stack<task_io_service>::contains(this))
+  if (thread_call_stack::contains(this))
   {
-    boost::asio::detail::fenced_block b;
+    fenced_block b(fenced_block::full);
     boost_asio_handler_invoke_helpers::invoke(handler, handler);
   }
   else

Modified: trunk/boost/asio/detail/impl/task_io_service.ipp
==============================================================================
--- trunk/boost/asio/detail/impl/task_io_service.ipp	(original)
+++ trunk/boost/asio/detail/impl/task_io_service.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -20,7 +20,6 @@
 #if !defined(BOOST_ASIO_HAS_IOCP)
 
 #include <boost/limits.hpp>
-#include <boost/asio/detail/call_stack.hpp>
 #include <boost/asio/detail/event.hpp>
 #include <boost/asio/detail/reactor.hpp>
 #include <boost/asio/detail/task_io_service.hpp>
@@ -48,24 +47,37 @@
   op_queue<operation>* ops_;
 };
 
-struct task_io_service::work_finished_on_block_exit
+struct task_io_service::work_cleanup
 {
-  ~work_finished_on_block_exit()
+  ~work_cleanup()
   {
     task_io_service_->work_finished();
+
+#if defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS) 
+    if (!ops_->empty())
+    {
+      lock_->lock();
+      task_io_service_->op_queue_.push(*ops_);
+    }
+#endif // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
   }
 
   task_io_service* task_io_service_;
+  mutex::scoped_lock* lock_;
+  op_queue<operation>* ops_;
 };
 
-struct task_io_service::idle_thread_info
+struct task_io_service::thread_info
 {
-  event wakeup_event;
-  idle_thread_info* next;
+  event* wakeup_event;
+  op_queue<operation>* private_op_queue;
+  thread_info* next;
 };
 
-task_io_service::task_io_service(boost::asio::io_service& io_service)
+task_io_service::task_io_service(
+    boost::asio::io_service& io_service, std::size_t concurrency_hint)
   : boost::asio::detail::service_base<task_io_service>(io_service),
+    one_thread_(concurrency_hint == 1),
     mutex_(),
     task_(0),
     task_interrupted_(true),
@@ -77,10 +89,6 @@
   BOOST_ASIO_HANDLER_TRACKING_INIT;
 }
 
-void task_io_service::init(std::size_t /*concurrency_hint*/)
-{
-}
-
 void task_io_service::shutdown_service()
 {
   mutex::scoped_lock lock(mutex_);
@@ -120,15 +128,22 @@
     return 0;
   }
 
-  call_stack<task_io_service>::context ctx(this);
-
-  idle_thread_info this_idle_thread;
-  this_idle_thread.next = 0;
+  thread_info this_thread;
+  event wakeup_event;
+  this_thread.wakeup_event = &wakeup_event;
+  op_queue<operation> private_op_queue;
+#if defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS) 
+  this_thread.private_op_queue = one_thread_ == 1 ? &private_op_queue : 0;
+#else // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+  this_thread.private_op_queue = 0;
+#endif // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+  this_thread.next = 0;
+  thread_call_stack::context ctx(this, this_thread);
 
   mutex::scoped_lock lock(mutex_);
 
   std::size_t n = 0;
-  for (; do_one(lock, &this_idle_thread); lock.lock())
+  for (; do_run_one(lock, this_thread, private_op_queue, ec); lock.lock())
     if (n != (std::numeric_limits<std::size_t>::max)())
       ++n;
   return n;
@@ -143,31 +158,53 @@
     return 0;
   }
 
-  call_stack<task_io_service>::context ctx(this);
-
-  idle_thread_info this_idle_thread;
-  this_idle_thread.next = 0;
+  thread_info this_thread;
+  event wakeup_event;
+  this_thread.wakeup_event = &wakeup_event;
+  op_queue<operation> private_op_queue;
+  this_thread.private_op_queue = 0;
+  this_thread.next = 0;
+  thread_call_stack::context ctx(this, this_thread);
 
   mutex::scoped_lock lock(mutex_);
 
-  return do_one(lock, &this_idle_thread);
+  return do_run_one(lock, this_thread, private_op_queue, ec);
 }
 
 std::size_t task_io_service::poll(boost::system::error_code& ec)
 {
+  ec = boost::system::error_code();
   if (outstanding_work_ == 0)
   {
     stop();
-    ec = boost::system::error_code();
     return 0;
   }
 
-  call_stack<task_io_service>::context ctx(this);
+  thread_info this_thread;
+  this_thread.wakeup_event = 0;
+  op_queue<operation> private_op_queue;
+#if defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS) 
+  this_thread.private_op_queue = one_thread_ == 1 ? &private_op_queue : 0;
+#else // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+  this_thread.private_op_queue = 0;
+#endif // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+  this_thread.next = 0;
+  thread_call_stack::context ctx(this, this_thread);
 
   mutex::scoped_lock lock(mutex_);
 
+#if defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS) 
+  // We want to support nested calls to poll() and poll_one(), so any handlers
+  // that are already on a thread-private queue need to be put on to the main
+  // queue now.
+  if (one_thread_)
+    if (thread_info* outer_thread_info = ctx.next_by_key())
+      if (outer_thread_info->private_op_queue)
+        op_queue_.push(*outer_thread_info->private_op_queue);
+#endif // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+
   std::size_t n = 0;
-  for (; do_one(lock, 0); lock.lock())
+  for (; do_poll_one(lock, private_op_queue, ec); lock.lock())
     if (n != (std::numeric_limits<std::size_t>::max)())
       ++n;
   return n;
@@ -182,11 +219,26 @@
     return 0;
   }
 
-  call_stack<task_io_service>::context ctx(this);
+  thread_info this_thread;
+  this_thread.wakeup_event = 0;
+  op_queue<operation> private_op_queue;
+  this_thread.private_op_queue = 0;
+  this_thread.next = 0;
+  thread_call_stack::context ctx(this, this_thread);
 
   mutex::scoped_lock lock(mutex_);
 
-  return do_one(lock, 0);
+#if defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS) 
+  // We want to support nested calls to poll() and poll_one(), so any handlers
+  // that are already on a thread-private queue need to be put on to the main
+  // queue now.
+  if (one_thread_)
+    if (thread_info* outer_thread_info = ctx.next_by_key())
+      if (outer_thread_info->private_op_queue)
+        op_queue_.push(*outer_thread_info->private_op_queue);
+#endif // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+
+  return do_poll_one(lock, private_op_queue, ec);
 }
 
 void task_io_service::stop()
@@ -215,6 +267,20 @@
 
 void task_io_service::post_deferred_completion(task_io_service::operation* op)
 {
+#if defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS) 
+  if (one_thread_)
+  {
+    if (thread_info* this_thread = thread_call_stack::contains(this))
+    {
+      if (this_thread->private_op_queue)
+      {
+        this_thread->private_op_queue->push(op);
+        return;
+      }
+    }
+  }
+#endif // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+
   mutex::scoped_lock lock(mutex_);
   op_queue_.push(op);
   wake_one_thread_and_unlock(lock);
@@ -225,6 +291,20 @@
 {
   if (!ops.empty())
   {
+#if defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS) 
+    if (one_thread_)
+    {
+      if (thread_info* this_thread = thread_call_stack::contains(this))
+      {
+        if (this_thread->private_op_queue)
+        {
+          this_thread->private_op_queue->push(ops);
+          return;
+        }
+      }
+    }
+#endif // defined(BOOST_HAS_THREADS) && !defined(BOOST_ASIO_DISABLE_THREADS)
+
     mutex::scoped_lock lock(mutex_);
     op_queue_.push(ops);
     wake_one_thread_and_unlock(lock);
@@ -238,11 +318,10 @@
   ops2.push(ops);
 }
 
-std::size_t task_io_service::do_one(mutex::scoped_lock& lock,
-    task_io_service::idle_thread_info* this_idle_thread)
+std::size_t task_io_service::do_run_one(mutex::scoped_lock& lock,
+    task_io_service::thread_info& this_thread,
+    op_queue<operation>& private_op_queue, const boost::system::error_code& ec)
 {
-  bool polling = !this_idle_thread;
-  bool task_has_run = false;
   while (!stopped_)
   {
     if (!op_queue_.empty())
@@ -254,61 +333,103 @@
 
       if (o == &task_operation_)
       {
-        task_interrupted_ = more_handlers || polling;
+        task_interrupted_ = more_handlers;
 
-        // If the task has already run and we're polling then we're done.
-        if (task_has_run && polling)
-        {
-          task_interrupted_ = true;
-          op_queue_.push(&task_operation_);
-          return 0;
-        }
-        task_has_run = true;
-
-        if (!more_handlers || !wake_one_idle_thread_and_unlock(lock))
+        if (more_handlers && !one_thread_)
+          wake_one_idle_thread_and_unlock(lock);
+        else
           lock.unlock();
 
         op_queue<operation> completed_ops;
-        task_cleanup c = { this, &lock, &completed_ops };
-        (void)c;
+        task_cleanup on_exit = { this, &lock, &completed_ops };
+        (void)on_exit;
 
         // Run the task. May throw an exception. Only block if the operation
         // queue is empty and we're not polling, otherwise we want to return
         // as soon as possible.
-        task_->run(!more_handlers && !polling, completed_ops);
+        task_->run(!more_handlers, completed_ops);
       }
       else
       {
-        if (more_handlers)
+        std::size_t task_result = o->task_result_;
+
+        if (more_handlers && !one_thread_)
           wake_one_thread_and_unlock(lock);
         else
           lock.unlock();
 
         // Ensure the count of outstanding work is decremented on block exit.
-        work_finished_on_block_exit on_exit = { this };
+        work_cleanup on_exit = { this, &lock, &private_op_queue };
         (void)on_exit;
 
-        // Complete the operation. May throw an exception.
-        o->complete(*this); // deletes the operation object
+        // Complete the operation. May throw an exception. Deletes the object.
+        o->complete(*this, ec, task_result);
 
         return 1;
       }
     }
-    else if (this_idle_thread)
+    else
     {
       // Nothing to run right now, so just wait for work to do.
-      this_idle_thread->next = first_idle_thread_;
-      first_idle_thread_ = this_idle_thread;
-      this_idle_thread->wakeup_event.clear(lock);
-      this_idle_thread->wakeup_event.wait(lock);
+      this_thread.next = first_idle_thread_;
+      first_idle_thread_ = &this_thread;
+      this_thread.wakeup_event->clear(lock);
+      this_thread.wakeup_event->wait(lock);
     }
-    else
+  }
+
+  return 0;
+}
+
+std::size_t task_io_service::do_poll_one(mutex::scoped_lock& lock,
+    op_queue<operation>& private_op_queue, const boost::system::error_code& ec)
+{
+  if (stopped_)
+    return 0;
+
+  operation* o = op_queue_.front();
+  if (o == &task_operation_)
+  {
+    op_queue_.pop();
+    lock.unlock();
+
     {
-      return 0;
+      op_queue<operation> completed_ops;
+      task_cleanup c = { this, &lock, &completed_ops };
+      (void)c;
+
+      // Run the task. May throw an exception. Only block if the operation
+      // queue is empty and we're not polling, otherwise we want to return
+      // as soon as possible.
+      task_->run(false, completed_ops);
     }
+
+    o = op_queue_.front();
+    if (o == &task_operation_)
+      return 0;
   }
 
-  return 0;
+  if (o == 0)
+    return 0;
+
+  op_queue_.pop();
+  bool more_handlers = (!op_queue_.empty());
+
+  std::size_t task_result = o->task_result_;
+
+  if (more_handlers && !one_thread_)
+    wake_one_thread_and_unlock(lock);
+  else
+    lock.unlock();
+
+  // Ensure the count of outstanding work is decremented on block exit.
+  work_cleanup on_exit = { this, &lock, &private_op_queue };
+  (void)on_exit;
+
+  // Complete the operation. May throw an exception. Deletes the object.
+  o->complete(*this, ec, task_result);
+
+  return 1;
 }
 
 void task_io_service::stop_all_threads(
@@ -318,10 +439,10 @@
 
   while (first_idle_thread_)
   {
-    idle_thread_info* idle_thread = first_idle_thread_;
+    thread_info* idle_thread = first_idle_thread_;
     first_idle_thread_ = idle_thread->next;
     idle_thread->next = 0;
-    idle_thread->wakeup_event.signal(lock);
+    idle_thread->wakeup_event->signal(lock);
   }
 
   if (!task_interrupted_ && task_)
@@ -336,10 +457,10 @@
 {
   if (first_idle_thread_)
   {
-    idle_thread_info* idle_thread = first_idle_thread_;
+    thread_info* idle_thread = first_idle_thread_;
     first_idle_thread_ = idle_thread->next;
     idle_thread->next = 0;
-    idle_thread->wakeup_event.signal_and_unlock(lock);
+    idle_thread->wakeup_event->signal_and_unlock(lock);
     return true;
   }
   return false;

Modified: trunk/boost/asio/detail/impl/win_iocp_io_service.hpp
==============================================================================
--- trunk/boost/asio/detail/impl/win_iocp_io_service.hpp	(original)
+++ trunk/boost/asio/detail/impl/win_iocp_io_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -19,7 +19,6 @@
 
 #if defined(BOOST_ASIO_HAS_IOCP)
 
-#include <boost/asio/detail/call_stack.hpp>
 #include <boost/asio/detail/completion_handler.hpp>
 #include <boost/asio/detail/fenced_block.hpp>
 #include <boost/asio/detail/handler_alloc_helpers.hpp>
@@ -36,7 +35,7 @@
 {
   if (call_stack<win_iocp_io_service>::contains(this))
   {
-    boost::asio::detail::fenced_block b;
+    fenced_block b(fenced_block::full);
     boost_asio_handler_invoke_helpers::invoke(handler, handler);
   }
   else

Modified: trunk/boost/asio/detail/impl/win_iocp_io_service.ipp
==============================================================================
--- trunk/boost/asio/detail/impl/win_iocp_io_service.ipp	(original)
+++ trunk/boost/asio/detail/impl/win_iocp_io_service.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -62,7 +62,8 @@
   win_iocp_io_service* io_service_;
 };
 
-win_iocp_io_service::win_iocp_io_service(boost::asio::io_service& io_service)
+win_iocp_io_service::win_iocp_io_service(
+    boost::asio::io_service& io_service, size_t concurrency_hint)
   : boost::asio::detail::service_base<win_iocp_io_service>(io_service),
     iocp_(),
     outstanding_work_(0),
@@ -71,10 +72,7 @@
     dispatch_required_(0)
 {
   BOOST_ASIO_HANDLER_TRACKING_INIT;
-}
 
-void win_iocp_io_service::init(size_t concurrency_hint)
-{
   iocp_.handle = ::CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0,
       static_cast<DWORD>((std::min<size_t>)(concurrency_hint, DWORD(~0))));
   if (!iocp_.handle)

Modified: trunk/boost/asio/detail/kqueue_reactor.hpp
==============================================================================
--- trunk/boost/asio/detail/kqueue_reactor.hpp	(original)
+++ trunk/boost/asio/detail/kqueue_reactor.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -62,12 +62,16 @@
   {
     friend class kqueue_reactor;
     friend class object_pool_access;
+
+    descriptor_state* next_;
+    descriptor_state* prev_;
+
+    bool op_queue_is_empty_[max_ops];
+
     mutex mutex_;
     int descriptor_;
     op_queue<reactor_op> op_queue_[max_ops];
     bool shutdown_;
-    descriptor_state* next_;
-    descriptor_state* prev_;
   };
 
   // Per-descriptor data.

Modified: trunk/boost/asio/detail/macos_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/macos_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/macos_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -31,8 +31,16 @@
   : private noncopyable
 {
 public:
-  // Constructor.
-  macos_fenced_block()
+  enum half_t { half };
+  enum full_t { full };
+
+  // Constructor for a half fenced block.
+  explicit macos_fenced_block(half_t)
+  {
+  }
+
+  // Constructor for a full fenced block.
+  explicit macos_fenced_block(full_t)
   {
     OSMemoryBarrier();
   }

Modified: trunk/boost/asio/detail/null_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/null_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/null_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -25,8 +25,10 @@
   : private noncopyable
 {
 public:
+  enum half_or_full_t { half, full };
+
   // Constructor.
-  null_fenced_block()
+  explicit null_fenced_block(half_or_full_t)
   {
   }
 

Modified: trunk/boost/asio/detail/reactive_null_buffers_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_null_buffers_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_null_buffers_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -47,7 +47,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_null_buffers_op* o(static_cast<reactive_null_buffers_op*>(base));
@@ -69,7 +70,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/reactive_socket_accept_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_socket_accept_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_socket_accept_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -95,7 +95,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_socket_accept_op* o(static_cast<reactive_socket_accept_op*>(base));
@@ -117,7 +118,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/reactive_socket_connect_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_socket_connect_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_socket_connect_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -64,7 +64,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_socket_connect_op* o
@@ -87,7 +88,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/reactive_socket_recv_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_socket_recv_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_socket_recv_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -82,7 +82,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_socket_recv_op* o(static_cast<reactive_socket_recv_op*>(base));
@@ -104,7 +105,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/reactive_socket_recvfrom_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_socket_recvfrom_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_socket_recvfrom_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -91,7 +91,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_socket_recvfrom_op* o(
@@ -114,7 +115,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/reactive_socket_recvmsg_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_socket_recvmsg_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_socket_recvmsg_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -83,7 +83,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_socket_recvmsg_op* o(
@@ -106,7 +107,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/reactive_socket_send_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_socket_send_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_socket_send_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -79,7 +79,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_socket_send_op* o(static_cast<reactive_socket_send_op*>(base));
@@ -101,7 +102,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/reactive_socket_sendto_op.hpp
==============================================================================
--- trunk/boost/asio/detail/reactive_socket_sendto_op.hpp	(original)
+++ trunk/boost/asio/detail/reactive_socket_sendto_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -82,7 +82,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     reactive_socket_sendto_op* o(static_cast<reactive_socket_sendto_op*>(base));
@@ -104,7 +105,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/resolve_endpoint_op.hpp
==============================================================================
--- trunk/boost/asio/detail/resolve_endpoint_op.hpp	(original)
+++ trunk/boost/asio/detail/resolve_endpoint_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -53,7 +53,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the operation object.
     resolve_endpoint_op* o(static_cast<resolve_endpoint_op*>(base));
@@ -96,7 +97,7 @@
 
       if (owner)
       {
-        boost::asio::detail::fenced_block b;
+        fenced_block b(fenced_block::half);
         BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, "..."));
         boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
         BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/resolve_op.hpp
==============================================================================
--- trunk/boost/asio/detail/resolve_op.hpp	(original)
+++ trunk/boost/asio/detail/resolve_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -61,7 +61,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the operation object.
     resolve_op* o(static_cast<resolve_op*>(base));
@@ -106,7 +107,7 @@
 
       if (owner)
       {
-        boost::asio::detail::fenced_block b;
+        fenced_block b(fenced_block::half);
         BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, "..."));
         boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
         BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/service_registry.hpp
==============================================================================
--- trunk/boost/asio/detail/service_registry.hpp	(original)
+++ trunk/boost/asio/detail/service_registry.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -52,8 +52,10 @@
   : private noncopyable
 {
 public:
-  // Constructor.
-  BOOST_ASIO_DECL service_registry(boost::asio::io_service& o);
+  // Constructor. Adds the initial service.
+  template <typename Service, typename Arg>
+  service_registry(boost::asio::io_service& o,
+      Service* initial_service, Arg arg);
 
   // Destructor.
   BOOST_ASIO_DECL ~service_registry();
@@ -61,6 +63,11 @@
   // Notify all services of a fork event.
   BOOST_ASIO_DECL void notify_fork(boost::asio::io_service::fork_event fork_ev);
 
+  // Get the first service object cast to the specified type. Called during
+  // io_service construction and so performs no locking or type checking.
+  template <typename Service>
+  Service& first_service();
+
   // Get the service object corresponding to the specified service type. Will
   // create a new service object automatically if no such object already
   // exists. Ownership of the service object is not transferred to the caller.
@@ -123,8 +130,8 @@
       const boost::asio::io_service::service::key& key,
       factory_type factory);
 
-  // Add a service object. Returns false on error, in which case ownership of
-  // the object is retained by the caller.
+  // Add a service object. Throws on error, in which case ownership of the
+  // object is retained by the caller.
   BOOST_ASIO_DECL void do_add_service(
       const boost::asio::io_service::service::key& key,
       boost::asio::io_service::service* new_service);

Modified: trunk/boost/asio/detail/signal_handler.hpp
==============================================================================
--- trunk/boost/asio/detail/signal_handler.hpp	(original)
+++ trunk/boost/asio/detail/signal_handler.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -40,7 +40,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     signal_handler* h(static_cast<signal_handler*>(base));
@@ -62,7 +63,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/signal_set_service.hpp
==============================================================================
--- trunk/boost/asio/detail/signal_set_service.hpp	(original)
+++ trunk/boost/asio/detail/signal_set_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -17,8 +17,8 @@
 
 #include <boost/asio/detail/config.hpp>
 
-#include <csignal>
 #include <cstddef>
+#include <signal.h>
 #include <boost/asio/error.hpp>
 #include <boost/asio/io_service.hpp>
 #include <boost/asio/detail/handler_alloc_helpers.hpp>

Modified: trunk/boost/asio/detail/solaris_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/solaris_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/solaris_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -31,8 +31,16 @@
   : private noncopyable
 {
 public:
-  // Constructor.
-  solaris_fenced_block()
+  enum half_t { half };
+  enum full_t { full };
+
+  // Constructor for a half fenced block.
+  explicit solaris_fenced_block(half_t)
+  {
+  }
+
+  // Constructor for a full fenced block.
+  explicit solaris_fenced_block(full_t)
   {
     membar_consumer();
   }

Modified: trunk/boost/asio/detail/strand_service.hpp
==============================================================================
--- trunk/boost/asio/detail/strand_service.hpp	(original)
+++ trunk/boost/asio/detail/strand_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -95,7 +95,7 @@
   BOOST_ASIO_DECL void do_post(implementation_type& impl, operation* op);
 
   BOOST_ASIO_DECL static void do_complete(io_service_impl* owner,
-      operation* base, boost::system::error_code ec,
+      operation* base, const boost::system::error_code& ec,
       std::size_t bytes_transferred);
 
   // The io_service implementation used to post completions.

Modified: trunk/boost/asio/detail/task_io_service.hpp
==============================================================================
--- trunk/boost/asio/detail/task_io_service.hpp	(original)
+++ trunk/boost/asio/detail/task_io_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -22,6 +22,7 @@
 #include <boost/system/error_code.hpp>
 #include <boost/asio/io_service.hpp>
 #include <boost/asio/detail/atomic_count.hpp>
+#include <boost/asio/detail/call_stack.hpp>
 #include <boost/asio/detail/mutex.hpp>
 #include <boost/asio/detail/op_queue.hpp>
 #include <boost/asio/detail/reactor_fwd.hpp>
@@ -40,11 +41,10 @@
 public:
   typedef task_io_service_operation operation;
 
-  // Constructor.
-  BOOST_ASIO_DECL task_io_service(boost::asio::io_service& io_service);
-
-  // How many concurrent threads are likely to run the io_service.
-  BOOST_ASIO_DECL void init(std::size_t concurrency_hint);
+  // Constructor. Specifies the number of concurrent threads that are likely to
+  // run the io_service. If set to 1 certain optimisation are performed.
+  BOOST_ASIO_DECL task_io_service(boost::asio::io_service& io_service,
+      std::size_t concurrency_hint = 0);
 
   // Destroy all user-defined handler objects owned by the service.
   BOOST_ASIO_DECL void shutdown_service();
@@ -86,6 +86,12 @@
       stop();
   }
 
+  // Return whether a handler can be dispatched immediately.
+  bool can_dispatch()
+  {
+    return thread_call_stack::contains(this);
+  }
+
   // Request invocation of the given handler.
   template <typename Handler>
   void dispatch(Handler handler);
@@ -112,11 +118,17 @@
 
 private:
   // Structure containing information about an idle thread.
-  struct idle_thread_info;
+  struct thread_info;
 
   // Run at most one operation. Blocks only if this_idle_thread is non-null.
-  BOOST_ASIO_DECL std::size_t do_one(mutex::scoped_lock& lock,
-      idle_thread_info* this_idle_thread);
+  BOOST_ASIO_DECL std::size_t do_run_one(mutex::scoped_lock& lock,
+      thread_info& this_thread, op_queue<operation>& private_op_queue,
+      const boost::system::error_code& ec);
+
+  // Poll for at most one operation.
+  BOOST_ASIO_DECL std::size_t do_poll_one(mutex::scoped_lock& lock,
+      op_queue<operation>& private_op_queue,
+      const boost::system::error_code& ec);
 
   // Stop the task and all idle threads.
   BOOST_ASIO_DECL void stop_all_threads(mutex::scoped_lock& lock);
@@ -135,8 +147,12 @@
   struct task_cleanup;
   friend struct task_cleanup;
 
-  // Helper class to call work_finished() on block exit.
-  struct work_finished_on_block_exit;
+  // Helper class to call work-related operations on block exit.
+  struct work_cleanup;
+  friend struct work_cleanup;
+
+  // Whether to optimise for single-threaded use cases.
+  const bool one_thread_;
 
   // Mutex to protect access to internal data.
   mutable mutex mutex_;
@@ -165,8 +181,11 @@
   // Flag to indicate that the dispatcher has been shut down.
   bool shutdown_;
 
+  // Per-thread call stack to track the state of each thread in the io_service.
+  typedef call_stack<task_io_service, thread_info> thread_call_stack;
+
   // The threads that are currently idle.
-  idle_thread_info* first_idle_thread_;
+  thread_info* first_idle_thread_;
 };
 
 } // namespace detail

Modified: trunk/boost/asio/detail/task_io_service_operation.hpp
==============================================================================
--- trunk/boost/asio/detail/task_io_service_operation.hpp	(original)
+++ trunk/boost/asio/detail/task_io_service_operation.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -31,9 +31,10 @@
 class task_io_service_operation BOOST_ASIO_INHERIT_TRACKED_HANDLER
 {
 public:
-  void complete(task_io_service& owner)
+  void complete(task_io_service& owner,
+      const boost::system::error_code& ec, std::size_t bytes_transferred)
   {
-    func_(&owner, this, boost::system::error_code(), 0);
+    func_(&owner, this, ec, bytes_transferred);
   }
 
   void destroy()
@@ -44,11 +45,12 @@
 protected:
   typedef void (*func_type)(task_io_service*,
       task_io_service_operation*,
-      boost::system::error_code, std::size_t);
+      const boost::system::error_code&, std::size_t);
 
   task_io_service_operation(func_type func)
     : next_(0),
-      func_(func)
+      func_(func),
+      task_result_(0)
   {
   }
 
@@ -61,6 +63,9 @@
   friend class op_queue_access;
   task_io_service_operation* next_;
   func_type func_;
+protected:
+  friend class task_io_service;
+  unsigned int task_result_; // Passed into bytes transferred.
 };
 
 } // namespace detail

Modified: trunk/boost/asio/detail/wait_handler.hpp
==============================================================================
--- trunk/boost/asio/detail/wait_handler.hpp	(original)
+++ trunk/boost/asio/detail/wait_handler.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -40,7 +40,8 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code /*ec*/, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& /*ec*/,
+      std::size_t /*bytes_transferred*/)
   {
     // Take ownership of the handler object.
     wait_handler* h(static_cast<wait_handler*>(base));
@@ -62,7 +63,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_fenced_block.hpp
==============================================================================
--- trunk/boost/asio/detail/win_fenced_block.hpp	(original)
+++ trunk/boost/asio/detail/win_fenced_block.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -31,8 +31,16 @@
   : private noncopyable
 {
 public:
-  // Constructor.
-  win_fenced_block()
+  enum half_t { half };
+  enum full_t { full };
+
+  // Constructor for a half fenced block.
+  explicit win_fenced_block(half_t)
+  {
+  }
+
+  // Constructor for a full fenced block.
+  explicit win_fenced_block(full_t)
   {
 #if defined(__BORLANDC__)
     LONG barrier = 0;

Modified: trunk/boost/asio/detail/win_iocp_handle_read_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_handle_read_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_handle_read_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -50,8 +50,11 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& result_ec,
+      std::size_t bytes_transferred)
   {
+    boost::system::error_code ec(result_ec);
+
     // Take ownership of the operation object.
     win_iocp_handle_read_op* o(static_cast<win_iocp_handle_read_op*>(base));
     ptr p = { boost::addressof(o->handler_), o, o };
@@ -85,7 +88,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_handle_write_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_handle_write_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_handle_write_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -49,7 +49,7 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& ec, std::size_t bytes_transferred)
   {
     // Take ownership of the operation object.
     win_iocp_handle_write_op* o(static_cast<win_iocp_handle_write_op*>(base));
@@ -80,7 +80,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_io_service.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_io_service.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_io_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -21,6 +21,7 @@
 
 #include <boost/limits.hpp>
 #include <boost/asio/io_service.hpp>
+#include <boost/asio/detail/call_stack.hpp>
 #include <boost/asio/detail/mutex.hpp>
 #include <boost/asio/detail/op_queue.hpp>
 #include <boost/asio/detail/scoped_ptr.hpp>
@@ -45,10 +46,11 @@
   : public boost::asio::detail::service_base<win_iocp_io_service>
 {
 public:
-  // Constructor.
-  BOOST_ASIO_DECL win_iocp_io_service(boost::asio::io_service& io_service);
 
-  BOOST_ASIO_DECL void init(size_t concurrency_hint);
+  // Constructor. Specifies a concurrency hint that is passed through to the
+  // underlying I/O completion port.
+  BOOST_ASIO_DECL win_iocp_io_service(boost::asio::io_service& io_service,
+      size_t concurrency_hint = 0);
 
   // Destroy all user-defined handler objects owned by the service.
   BOOST_ASIO_DECL void shutdown_service();
@@ -102,6 +104,12 @@
       stop();
   }
 
+  // Return whether a handler can be dispatched immediately.
+  bool can_dispatch()
+  {
+    return call_stack<win_iocp_io_service>::contains(this) != 0;
+  }
+
   // Request invocation of the given handler.
   template <typename Handler>
   void dispatch(Handler handler);

Modified: trunk/boost/asio/detail/win_iocp_null_buffers_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_null_buffers_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_null_buffers_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -56,8 +56,11 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& result_ec,
+      std::size_t bytes_transferred)
   {
+    boost::system::error_code ec(result_ec);
+
     // Take ownership of the operation object.
     win_iocp_null_buffers_op* o(static_cast<win_iocp_null_buffers_op*>(base));
     ptr p = { boost::addressof(o->handler_), o, o };
@@ -95,7 +98,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_operation.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_operation.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_operation.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -38,8 +38,8 @@
 {
 public:
   void complete(win_iocp_io_service& owner,
-      const boost::system::error_code& ec = boost::system::error_code(),
-      std::size_t bytes_transferred = 0)
+      const boost::system::error_code& ec,
+      std::size_t bytes_transferred)
   {
     func_(&owner, this, ec, bytes_transferred);
   }
@@ -50,8 +50,9 @@
   }
 
 protected:
-  typedef void (*func_type)(win_iocp_io_service*,
-      win_iocp_operation*, boost::system::error_code, std::size_t);
+  typedef void (*func_type)(
+      win_iocp_io_service*, win_iocp_operation*,
+      const boost::system::error_code&, std::size_t);
 
   win_iocp_operation(func_type func)
     : next_(0),

Modified: trunk/boost/asio/detail/win_iocp_overlapped_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_overlapped_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_overlapped_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -46,7 +46,7 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& ec, std::size_t bytes_transferred)
   {
     // Take ownership of the operation object.
     win_iocp_overlapped_op* o(static_cast<win_iocp_overlapped_op*>(base));
@@ -68,7 +68,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_socket_accept_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_socket_accept_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_socket_accept_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -73,8 +73,11 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t /*bytes_transferred*/)
+      const boost::system::error_code& result_ec,
+      std::size_t /*bytes_transferred*/)
   {
+    boost::system::error_code ec(result_ec);
+
     // Take ownership of the operation object.
     win_iocp_socket_accept_op* o(static_cast<win_iocp_socket_accept_op*>(base));
     ptr p = { boost::addressof(o->handler_), o, o };
@@ -134,7 +137,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_socket_recv_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_socket_recv_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_socket_recv_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -53,8 +53,11 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& result_ec,
+      std::size_t bytes_transferred)
   {
+    boost::system::error_code ec(result_ec);
+
     // Take ownership of the operation object.
     win_iocp_socket_recv_op* o(static_cast<win_iocp_socket_recv_op*>(base));
     ptr p = { boost::addressof(o->handler_), o, o };
@@ -89,7 +92,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_socket_recvfrom_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_socket_recvfrom_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_socket_recvfrom_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -59,8 +59,11 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& result_ec,
+      std::size_t bytes_transferred)
   {
+    boost::system::error_code ec(result_ec);
+
     // Take ownership of the operation object.
     win_iocp_socket_recvfrom_op* o(
         static_cast<win_iocp_socket_recvfrom_op*>(base));
@@ -96,7 +99,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_socket_recvmsg_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_socket_recvmsg_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_socket_recvmsg_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -55,8 +55,11 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& result_ec,
+      std::size_t bytes_transferred)
   {
+    boost::system::error_code ec(result_ec);
+
     // Take ownership of the operation object.
     win_iocp_socket_recvmsg_op* o(
         static_cast<win_iocp_socket_recvmsg_op*>(base));
@@ -90,7 +93,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/detail/win_iocp_socket_send_op.hpp
==============================================================================
--- trunk/boost/asio/detail/win_iocp_socket_send_op.hpp	(original)
+++ trunk/boost/asio/detail/win_iocp_socket_send_op.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -51,8 +51,11 @@
   }
 
   static void do_complete(io_service_impl* owner, operation* base,
-      boost::system::error_code ec, std::size_t bytes_transferred)
+      const boost::system::error_code& result_ec,
+      std::size_t bytes_transferred)
   {
+    boost::system::error_code ec(result_ec);
+
     // Take ownership of the operation object.
     win_iocp_socket_send_op* o(static_cast<win_iocp_socket_send_op*>(base));
     ptr p = { boost::addressof(o->handler_), o, o };
@@ -84,7 +87,7 @@
     // Make the upcall if required.
     if (owner)
     {
-      boost::asio::detail::fenced_block b;
+      fenced_block b(fenced_block::half);
       BOOST_ASIO_HANDLER_INVOCATION_BEGIN((handler.arg1_, handler.arg2_));
       boost_asio_handler_invoke_helpers::invoke(handler, handler.handler_);
       BOOST_ASIO_HANDLER_INVOCATION_END;

Modified: trunk/boost/asio/impl/io_service.hpp
==============================================================================
--- trunk/boost/asio/impl/io_service.hpp	(original)
+++ trunk/boost/asio/impl/io_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -33,6 +33,13 @@
   return ios.service_registry_->template use_service<Service>();
 }
 
+template <>
+inline detail::io_service_impl& use_service<detail::io_service_impl>(
+    io_service& ios)
+{
+  return ios.impl_;
+}
+
 template <typename Service>
 inline void add_service(io_service& ios, Service* svc)
 {
@@ -102,25 +109,25 @@
 }
 
 inline io_service::work::work(boost::asio::io_service& io_service)
-  : io_service_(io_service)
+  : io_service_impl_(io_service.impl_)
 {
-  io_service_.impl_.work_started();
+  io_service_impl_.work_started();
 }
 
 inline io_service::work::work(const work& other)
-  : io_service_(other.io_service_)
+  : io_service_impl_(other.io_service_impl_)
 {
-  io_service_.impl_.work_started();
+  io_service_impl_.work_started();
 }
 
 inline io_service::work::~work()
 {
-  io_service_.impl_.work_finished();
+  io_service_impl_.work_finished();
 }
 
 inline boost::asio::io_service& io_service::work::get_io_service()
 {
-  return io_service_;
+  return io_service_impl_.get_io_service();
 }
 
 inline boost::asio::io_service& io_service::service::get_io_service()

Modified: trunk/boost/asio/impl/io_service.ipp
==============================================================================
--- trunk/boost/asio/impl/io_service.ipp	(original)
+++ trunk/boost/asio/impl/io_service.ipp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -18,6 +18,7 @@
 #include <boost/asio/detail/config.hpp>
 #include <boost/limits.hpp>
 #include <boost/asio/io_service.hpp>
+#include <boost/asio/detail/scoped_ptr.hpp>
 #include <boost/asio/detail/service_registry.hpp>
 #include <boost/asio/detail/throw_error.hpp>
 
@@ -33,17 +34,18 @@
 namespace asio {
 
 io_service::io_service()
-  : service_registry_(new boost::asio::detail::service_registry(*this)),
-    impl_(service_registry_->use_service<impl_type>())
+  : service_registry_(new boost::asio::detail::service_registry(
+        *this, static_cast<impl_type*>(0),
+        (std::numeric_limits<std::size_t>::max)())),
+    impl_(service_registry_->first_service<impl_type>())
 {
-  impl_.init((std::numeric_limits<std::size_t>::max)());
 }
 
 io_service::io_service(std::size_t concurrency_hint)
-  : service_registry_(new boost::asio::detail::service_registry(*this)),
-    impl_(service_registry_->use_service<impl_type>())
+  : service_registry_(new boost::asio::detail::service_registry(
+        *this, static_cast<impl_type*>(0), concurrency_hint)),
+    impl_(service_registry_->first_service<impl_type>())
 {
-  impl_.init(concurrency_hint);
 }
 
 io_service::~io_service()

Modified: trunk/boost/asio/io_service.hpp
==============================================================================
--- trunk/boost/asio/io_service.hpp	(original)
+++ trunk/boost/asio/io_service.hpp	2011-10-08 17:58:10 EDT (Sat, 08 Oct 2011)
@@ -658,8 +658,8 @@
   // Prevent assignment.
   void operator=(const work& other);
 
-  // The io_service.
-  boost::asio::io_service& io_service_;
+  // The io_service implementation.
+  detail::io_service_impl& io_service_impl_;
 };
 
 /// Class used to uniquely identify a service.