[Boost-commit] svn:boost r82106 - trunk/boost/atomic/detail

Subject: [Boost-commit] svn:boost r82106 - trunk/boost/atomic/detail
From: andrey.semashev_at_[hidden]
Date: 2012-12-19 17:00:26

Author: andysem
Date: 2012-12-19 17:00:25 EST (Wed, 19 Dec 2012)
New Revision: 82106
URL: http://svn.boost.org/trac/boost/changeset/82106

Log:
Removed unnecessary fences, added pause operations to tight spin loops to optimize power consumption and performance on HT-enabled systems.
Text files modified: 
   trunk/boost/atomic/detail/gcc-x86.hpp  |   277 +++++++++++++++++++++---------------    
   trunk/boost/atomic/detail/lockpool.hpp |     8                                         
   trunk/boost/atomic/detail/windows.hpp  |   300 +++++++++------------------------------ 
   3 files changed, 244 insertions(+), 341 deletions(-)

Modified: trunk/boost/atomic/detail/gcc-x86.hpp
==============================================================================
--- trunk/boost/atomic/detail/gcc-x86.hpp	(original)
+++ trunk/boost/atomic/detail/gcc-x86.hpp	2012-12-19 17:00:25 EST (Wed, 19 Dec 2012)
@@ -20,128 +20,138 @@
 namespace atomics {
 namespace detail {
 
-#if defined(__x86_64__)
+#if defined(__x86_64__) || defined(__SSE2__)
 # define BOOST_ATOMIC_X86_FENCE_INSTR "mfence\n"
 #else
 # define BOOST_ATOMIC_X86_FENCE_INSTR "lock ; addl $0, (%%esp)\n"
 #endif
 
+#define BOOST_ATOMIC_X86_PAUSE() __asm__ __volatile__ ("pause\n")
+
 inline void
 platform_fence_before(memory_order order)
 {
-    switch(order) {
-        case memory_order_relaxed:
-        case memory_order_acquire:
-        case memory_order_consume:
-            break;
-        case memory_order_release:
-        case memory_order_acq_rel:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* release */
-            break;
-        case memory_order_seq_cst:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* seq */
-            break;
+    switch(order)
+    {
+    case memory_order_relaxed:
+    case memory_order_acquire:
+    case memory_order_consume:
+        break;
+    case memory_order_release:
+    case memory_order_acq_rel:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* release */
+        break;
+    case memory_order_seq_cst:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* seq */
+        break;
+    default:;
     }
 }
 
 inline void
 platform_fence_after(memory_order order)
 {
-    switch(order) {
-        case memory_order_relaxed:
-        case memory_order_release:
-            break;
-        case memory_order_acquire:
-        case memory_order_acq_rel:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* acquire */
-            break;
-        case memory_order_consume:
-            /* consume */
-            break;
-        case memory_order_seq_cst:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* seq */
-            break;
-        default:;
+    switch(order)
+    {
+    case memory_order_relaxed:
+    case memory_order_release:
+        break;
+    case memory_order_acquire:
+    case memory_order_acq_rel:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* acquire */
+        break;
+    case memory_order_consume:
+        /* consume */
+        break;
+    case memory_order_seq_cst:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* seq */
+        break;
+    default:;
     }
 }
 
 inline void
 platform_fence_after_load(memory_order order)
 {
-    switch(order) {
-        case memory_order_relaxed:
-        case memory_order_release:
-            break;
-        case memory_order_acquire:
-        case memory_order_acq_rel:
-            __asm__ __volatile__ ("" ::: "memory");
-            break;
-        case memory_order_consume:
-            break;
-        case memory_order_seq_cst:
-            __asm__ __volatile__ (BOOST_ATOMIC_X86_FENCE_INSTR ::: "memory");
-            break;
-        default:;
+    switch(order)
+    {
+    case memory_order_relaxed:
+    case memory_order_release:
+        break;
+    case memory_order_acquire:
+    case memory_order_acq_rel:
+        __asm__ __volatile__ ("" ::: "memory");
+        break;
+    case memory_order_consume:
+        break;
+    case memory_order_seq_cst:
+        __asm__ __volatile__ ("" ::: "memory");
+        break;
+    default:;
     }
 }
 
 inline void
 platform_fence_before_store(memory_order order)
 {
-    switch(order) {
-        case memory_order_relaxed:
-        case memory_order_acquire:
-        case memory_order_consume:
-            break;
-        case memory_order_release:
-        case memory_order_acq_rel:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* release */
-            break;
-        case memory_order_seq_cst:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* seq */
-            break;
+    switch(order)
+    {
+    case memory_order_relaxed:
+    case memory_order_acquire:
+    case memory_order_consume:
+        break;
+    case memory_order_release:
+    case memory_order_acq_rel:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* release */
+        break;
+    case memory_order_seq_cst:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* seq */
+        break;
+    default:;
     }
 }
 
 inline void
 platform_fence_after_store(memory_order order)
 {
-    switch(order) {
-        case memory_order_relaxed:
-        case memory_order_release:
-            break;
-        case memory_order_acquire:
-        case memory_order_acq_rel:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* acquire */
-            break;
-        case memory_order_consume:
-            /* consume */
-            break;
-        case memory_order_seq_cst:
-            __asm__ __volatile__ ("" ::: "memory");
-            /* seq */
-            break;
-        default:;
+    switch(order)
+    {
+    case memory_order_relaxed:
+    case memory_order_release:
+        break;
+    case memory_order_acquire:
+    case memory_order_acq_rel:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* acquire */
+        break;
+    case memory_order_consume:
+        /* consume */
+        break;
+    case memory_order_seq_cst:
+        __asm__ __volatile__ ("" ::: "memory");
+        /* seq */
+        break;
+    default:;
     }
 }
 
 }
 }
 
-class atomic_flag {
+class atomic_flag
+{
 private:
     atomic_flag(const atomic_flag &) /* = delete */ ;
     atomic_flag & operator=(const atomic_flag &) /* = delete */ ;
     uint32_t v_;
 public:
-    atomic_flag(void) : v_(false) {}
+    atomic_flag(void) : v_(0) {}
 
     bool
     test_and_set(memory_order order = memory_order_seq_cst) volatile
@@ -203,24 +213,25 @@
 inline void
 atomic_thread_fence(memory_order order)
 {
-    switch(order) {
-        case memory_order_relaxed:
-            break;
-        case memory_order_release:
-            __asm__ __volatile__ ("" ::: "memory");
-            break;
-        case memory_order_acquire:
-            __asm__ __volatile__ ("" ::: "memory");
-            break;
-        case memory_order_acq_rel:
-            __asm__ __volatile__ ("" ::: "memory");
-            break;
-        case memory_order_consume:
-            break;
-        case memory_order_seq_cst:
-            __asm__ __volatile__ (BOOST_ATOMIC_X86_FENCE_INSTR ::: "memory");
-            break;
-        default:;
+    switch(order)
+    {
+    case memory_order_relaxed:
+        break;
+    case memory_order_release:
+        __asm__ __volatile__ ("" ::: "memory");
+        break;
+    case memory_order_acquire:
+        __asm__ __volatile__ ("" ::: "memory");
+        break;
+    case memory_order_acq_rel:
+        __asm__ __volatile__ ("" ::: "memory");
+        break;
+    case memory_order_consume:
+        break;
+    case memory_order_seq_cst:
+        __asm__ __volatile__ (BOOST_ATOMIC_X86_FENCE_INSTR ::: "memory");
+        break;
+    default:;
     }
 }
 
@@ -329,7 +340,10 @@
     fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for(; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -337,7 +351,10 @@
     fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -345,7 +362,10 @@
     fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -457,7 +477,10 @@
     fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -465,7 +488,10 @@
     fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -473,7 +499,10 @@
     fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -585,7 +614,10 @@
     fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -593,7 +625,10 @@
     fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -601,7 +636,10 @@
     fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -714,7 +752,10 @@
     fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -722,7 +763,10 @@
     fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -730,7 +774,10 @@
     fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile
     {
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
     }
 
@@ -1570,8 +1617,10 @@
 platform_store64(T value, volatile T * ptr)
 {
     T expected = *ptr;
-    do {
-    } while (!platform_cmpxchg64_strong(expected, value, ptr));
+    for (; !platform_cmpxchg64_strong(expected, value, ptr);)
+    {
+        BOOST_ATOMIC_X86_PAUSE();
+    }
 }
 
 template<typename T>
@@ -1579,8 +1628,10 @@
 platform_load64(const volatile T * ptr)
 {
     T expected = *ptr;
-    do {
-    } while (!platform_cmpxchg64_strong(expected, expected, const_cast<volatile T*>(ptr)));
+    for (; !platform_cmpxchg64_strong(expected, expected, const_cast<volatile T*>(ptr));)
+    {
+        BOOST_ATOMIC_X86_PAUSE();
+    }
     return expected;
 }
 

Modified: trunk/boost/atomic/detail/lockpool.hpp
==============================================================================
--- trunk/boost/atomic/detail/lockpool.hpp	(original)
+++ trunk/boost/atomic/detail/lockpool.hpp	2012-12-19 17:00:25 EST (Wed, 19 Dec 2012)
@@ -69,8 +69,12 @@
         explicit
         scoped_lock(const volatile void * addr) : flag_(get_lock_for(addr))
         {
-            do {
-            } while (flag_.test_and_set(memory_order_acquire));
+            for (; flag_.test_and_set(memory_order_acquire);)
+            {
+#if defined(BOOST_ATOMIC_X86_PAUSE)
+                BOOST_ATOMIC_X86_PAUSE();
+#endif
+            }
         }
 
         ~scoped_lock(void)

Modified: trunk/boost/atomic/detail/windows.hpp
==============================================================================
--- trunk/boost/atomic/detail/windows.hpp	(original)
+++ trunk/boost/atomic/detail/windows.hpp	2012-12-19 17:00:25 EST (Wed, 19 Dec 2012)
@@ -28,6 +28,14 @@
 namespace atomics {
 namespace detail {
 
+#if defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_IX86))
+extern "C" void _mm_pause(void);
+#pragma intrinsic(_mm_pause)
+#define BOOST_ATOMIC_X86_PAUSE() _mm_pause()
+#else
+#define BOOST_ATOMIC_X86_PAUSE()
+#endif
+
 // Define hardware barriers
 #if defined(_MSC_VER) && (defined(_M_AMD64) || (defined(_M_IX86) && defined(_M_IX86_FP) && _M_IX86_FP >= 2))
 extern "C" void _mm_mfence(void);
@@ -53,174 +61,40 @@
 
 #define BOOST_ATOMIC_READ_WRITE_BARRIER() _ReadWriteBarrier()
 
-#if _MSC_VER >= 1400
-
-extern "C" void _ReadBarrier();
-#pragma intrinsic(_ReadBarrier)
-extern "C" void _WriteBarrier();
-#pragma intrinsic(_WriteBarrier)
-
-#define BOOST_ATOMIC_READ_BARRIER() _ReadBarrier()
-#define BOOST_ATOMIC_WRITE_BARRIER() _WriteBarrier()
-
-#endif
 #endif
 
 #ifndef BOOST_ATOMIC_READ_WRITE_BARRIER
 #define BOOST_ATOMIC_READ_WRITE_BARRIER()
 #endif
-#ifndef BOOST_ATOMIC_READ_BARRIER
-#define BOOST_ATOMIC_READ_BARRIER() BOOST_ATOMIC_READ_WRITE_BARRIER()
-#endif
-#ifndef BOOST_ATOMIC_WRITE_BARRIER
-#define BOOST_ATOMIC_WRITE_BARRIER() BOOST_ATOMIC_READ_WRITE_BARRIER()
-#endif
-
-// MSVC (up to 2012, inclusively) optimizer generates a very poor code for switch-case in fence functions.
-// Issuing unconditional compiler barriers generates better code. We may re-enable the main branch if MSVC optimizer improves.
-#ifdef BOOST_MSVC
-#define BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-#endif
 
 BOOST_FORCEINLINE void
-platform_fence_before(memory_order order)
+platform_fence_before(memory_order)
 {
-#ifdef BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-
     BOOST_ATOMIC_READ_WRITE_BARRIER();
-
-#else
-
-    switch(order)
-    {
-    case memory_order_relaxed:
-    case memory_order_consume:
-    case memory_order_acquire:
-        break;
-    case memory_order_release:
-    case memory_order_acq_rel:
-        BOOST_ATOMIC_WRITE_BARRIER();
-        /* release */
-        break;
-    case memory_order_seq_cst:
-        BOOST_ATOMIC_READ_WRITE_BARRIER();
-        /* seq */
-        break;
-    }
-
-#endif
 }
 
 BOOST_FORCEINLINE void
-platform_fence_after(memory_order order)
+platform_fence_after(memory_order)
 {
-#ifdef BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-
     BOOST_ATOMIC_READ_WRITE_BARRIER();
-
-#else
-
-    switch(order)
-    {
-    case memory_order_relaxed:
-    case memory_order_release:
-        break;
-    case memory_order_consume:
-    case memory_order_acquire:
-    case memory_order_acq_rel:
-        BOOST_ATOMIC_READ_BARRIER();
-        break;
-    case memory_order_seq_cst:
-        BOOST_ATOMIC_READ_WRITE_BARRIER();
-        /* seq */
-        break;
-    }
-
-#endif
 }
 
 BOOST_FORCEINLINE void
-platform_fence_before_store(memory_order order)
+platform_fence_before_store(memory_order)
 {
-#ifdef BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-
-    BOOST_ATOMIC_WRITE_BARRIER();
-
-#else
-
-    switch(order)
-    {
-    case memory_order_relaxed:
-    case memory_order_acquire:
-    case memory_order_consume:
-        break;
-    case memory_order_acq_rel:
-    case memory_order_release:
-    case memory_order_seq_cst:
-        BOOST_ATOMIC_WRITE_BARRIER();
-        break;
-    }
-
-#endif
+    BOOST_ATOMIC_READ_WRITE_BARRIER();
 }
 
 BOOST_FORCEINLINE void
-platform_fence_after_store(memory_order order)
+platform_fence_after_store(memory_order)
 {
-#ifdef BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-
-    BOOST_ATOMIC_WRITE_BARRIER();
-    if (order == memory_order_seq_cst)
-        x86_full_fence();
-
-#else
-
-    switch(order)
-    {
-    case memory_order_relaxed:
-    case memory_order_acquire:
-    case memory_order_consume:
-        break;
-    case memory_order_acq_rel:
-    case memory_order_release:
-        BOOST_ATOMIC_WRITE_BARRIER();
-        break;
-    case memory_order_seq_cst:
-        x86_full_fence();
-        break;
-    }
-
-#endif
+    BOOST_ATOMIC_READ_WRITE_BARRIER();
 }
 
 BOOST_FORCEINLINE void
 platform_fence_after_load(memory_order order)
 {
-#ifdef BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-
-    BOOST_ATOMIC_READ_BARRIER();
-    if (order == memory_order_seq_cst)
-        x86_full_fence();
-
-#else
-
-    switch(order)
-    {
-    case memory_order_relaxed:
-    case memory_order_consume:
-        break;
-    case memory_order_acquire:
-    case memory_order_acq_rel:
-        BOOST_ATOMIC_READ_BARRIER();
-        break;
-    case memory_order_release:
-        break;
-    case memory_order_seq_cst:
-        x86_full_fence();
-        break;
-    }
-
-#endif
+    BOOST_ATOMIC_READ_WRITE_BARRIER();
 }
 
 } // namespace detail
@@ -230,107 +104,45 @@
 BOOST_FORCEINLINE void
 atomic_thread_fence(memory_order order)
 {
-#ifdef BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-
     BOOST_ATOMIC_READ_WRITE_BARRIER();
     if (order == memory_order_seq_cst)
         atomics::detail::x86_full_fence();
-
-#else
-
-    switch (order)
-    {
-    case memory_order_relaxed:
-        break;
-    case memory_order_consume:
-    case memory_order_acquire:
-        BOOST_ATOMIC_READ_BARRIER();
-        break;
-    case memory_order_release:
-        BOOST_ATOMIC_WRITE_BARRIER();
-        break;
-    case memory_order_acq_rel:
-        BOOST_ATOMIC_READ_WRITE_BARRIER();
-        break;
-    case memory_order_seq_cst:
-        atomics::detail::x86_full_fence();
-        break;
-    }
-
-#endif
 }
 
 #define BOOST_ATOMIC_SIGNAL_FENCE 2
 BOOST_FORCEINLINE void
-atomic_signal_fence(memory_order order)
+atomic_signal_fence(memory_order)
 {
-#ifdef BOOST_ATOMIC_DETAIL_BAD_SWITCH_CASE_OPTIMIZER
-
     BOOST_ATOMIC_READ_WRITE_BARRIER();
-
-#else
-
-    switch (order)
-    {
-    case memory_order_relaxed:
-        break;
-    case memory_order_consume:
-    case memory_order_acquire:
-        BOOST_ATOMIC_READ_BARRIER();
-        break;
-    case memory_order_release:
-        BOOST_ATOMIC_WRITE_BARRIER();
-        break;
-    case memory_order_acq_rel:
-    case memory_order_seq_cst:
-        BOOST_ATOMIC_READ_WRITE_BARRIER();
-        break;
-    }
-
-#endif
 }
 
 #undef BOOST_ATOMIC_READ_WRITE_BARRIER
-#undef BOOST_ATOMIC_READ_BARRIER
-#undef BOOST_ATOMIC_WRITE_BARRIER
 
 class atomic_flag
 {
 private:
     atomic_flag(const atomic_flag &) /* = delete */ ;
     atomic_flag & operator=(const atomic_flag &) /* = delete */ ;
-#ifdef BOOST_ATOMIC_INTERLOCKED_EXCHANGE8
-    char v_;
-#else
-    long v_;
-#endif
+    uint32_t v_;
 public:
     atomic_flag(void) : v_(0) {}
 
-    void
-    clear(memory_order order = memory_order_seq_cst) volatile
-    {
-        atomics::detail::platform_fence_before_store(order);
-#ifdef BOOST_ATOMIC_INTERLOCKED_EXCHANGE8
-        BOOST_ATOMIC_INTERLOCKED_EXCHANGE8(&v_, 0);
-#else
-        BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&v_, 0);
-#endif
-        atomics::detail::platform_fence_after_store(order);
-    }
-
     bool
     test_and_set(memory_order order = memory_order_seq_cst) volatile
     {
         atomics::detail::platform_fence_before(order);
-#ifdef BOOST_ATOMIC_INTERLOCKED_EXCHANGE8
-        const char old = BOOST_ATOMIC_INTERLOCKED_EXCHANGE8(&v_, 1);
-#else
-        const long old = BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&v_, 1);
-#endif
+        const uint32_t old = (uint32_t)BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&v_, 1);
         atomics::detail::platform_fence_after(order);
         return old != 0;
     }
+
+    void
+    clear(memory_order order = memory_order_seq_cst) volatile
+    {
+        atomics::detail::platform_fence_before_store(order);
+        BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&v_, 0);
+        atomics::detail::platform_fence_after_store(order);
+    }
 };
 
 } // namespace boost
@@ -478,7 +290,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -498,7 +313,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -518,7 +336,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -655,7 +476,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -675,7 +499,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -695,7 +522,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -807,7 +637,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -822,7 +655,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for(; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -837,7 +673,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -951,7 +790,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -966,7 +808,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }
@@ -981,7 +826,10 @@
         return v;
 #else
         value_type tmp = load(memory_order_relaxed);
-        do {} while(!compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed));
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
         return tmp;
 #endif
     }