[Boost-commit] svn:boost r84695 - in trunk: boost/atomic/detail libs/atomic/test

Subject: [Boost-commit] svn:boost r84695 - in trunk: boost/atomic/detail libs/atomic/test
From: andrey.semashev_at_[hidden]
Date: 2013-06-08 13:36:38

Author: andysem
Date: 2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)
New Revision: 84695
URL: http://svn.boost.org/trac/boost/changeset/84695

Log:
Added implementation of 64-bit atomic operations on 32-bit Windows. Pointer specializations of 64-bit CAS-based atomic implementation extracted to a separate header, since these specializations are not needed on 32-bit platforms.

Added:
   trunk/boost/atomic/detail/cas64strong-ptr.hpp   (contents, props changed)
Text files modified: 
   trunk/boost/atomic/detail/cas64strong-ptr.hpp |   232 ++++++++++++++++++++                    
   trunk/boost/atomic/detail/cas64strong.hpp     |   198 -----------------                       
   trunk/boost/atomic/detail/gcc-x86.hpp         |    21 +                                       
   trunk/boost/atomic/detail/interlocked.hpp     |     7                                         
   trunk/boost/atomic/detail/windows.hpp         |   447 +++++++++++++++++++++++++-------------- 
   trunk/libs/atomic/test/lockfree.cpp           |     2                                         
   6 files changed, 540 insertions(+), 367 deletions(-)

Added: trunk/boost/atomic/detail/cas64strong-ptr.hpp
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ trunk/boost/atomic/detail/cas64strong-ptr.hpp	2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)	(r84695)
@@ -0,0 +1,232 @@
+#ifndef BOOST_ATOMIC_DETAIL_CAS64STRONG_PTR_HPP
+#define BOOST_ATOMIC_DETAIL_CAS64STRONG_PTR_HPP
+
+//  Distributed under the Boost Software License, Version 1.0.
+//  See accompanying file LICENSE_1_0.txt or copy at
+//  http://www.boost.org/LICENSE_1_0.txt)
+//
+//  Copyright (c) 2011 Helge Bahmann
+//  Copyright (c) 2013 Tim Blechmann
+
+// Build 64-bit atomic operation on pointers from platform_cmpxchg64_strong
+// primitive. It is assumed that 64-bit loads/stores are not
+// atomic, so they are implemented through platform_load64/platform_store64.
+//
+// The reason for extracting pointer specializations to a separate header is
+// that 64-bit CAS is available on some 32-bit platforms (notably, x86).
+// On these platforms there is no need for 64-bit pointer specializations,
+// since they will never be used.
+
+#include <cstddef>
+#include <boost/cstdint.hpp>
+#include <boost/memory_order.hpp>
+#include <boost/atomic/detail/config.hpp>
+#include <boost/atomic/detail/base.hpp>
+
+#ifdef BOOST_ATOMIC_HAS_PRAGMA_ONCE
+#pragma once
+#endif
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+/* pointer types */
+
+template<bool Sign>
+class base_atomic<void *, void *, 8, Sign>
+{
+    typedef base_atomic this_type;
+    typedef void * value_type;
+    typedef ptrdiff_t difference_type;
+public:
+    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
+    base_atomic(void) {}
+
+    void
+    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        platform_fence_before_store(order);
+        platform_store64(v, &v_);
+        platform_fence_after_store(order);
+    }
+
+    value_type
+    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
+    {
+        value_type v = platform_load64(&v_);
+        platform_fence_after_load(order);
+        return v;
+    }
+
+    value_type
+    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        value_type original = load(memory_order_relaxed);
+        do {
+        } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
+        return original;
+    }
+
+    bool
+    compare_exchange_weak(
+        value_type & expected,
+        value_type desired,
+        memory_order success_order,
+        memory_order failure_order) volatile BOOST_NOEXCEPT
+    {
+        return compare_exchange_strong(expected, desired, success_order, failure_order);
+    }
+
+    bool
+    compare_exchange_strong(
+        value_type & expected,
+        value_type desired,
+        memory_order success_order,
+        memory_order failure_order) volatile BOOST_NOEXCEPT
+    {
+        platform_fence_before(success_order);
+
+        bool success = platform_cmpxchg64_strong(expected, desired, &v_);
+
+        if (success) {
+            platform_fence_after(success_order);
+        } else {
+            platform_fence_after(failure_order);
+        }
+
+        return success;
+    }
+
+    value_type
+    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        value_type original = load(memory_order_relaxed);
+        do {
+        } while (!compare_exchange_weak(original, (char*)original + v, order, memory_order_relaxed));
+        return original;
+    }
+
+    value_type
+    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        value_type original = load(memory_order_relaxed);
+        do {
+        } while (!compare_exchange_weak(original, (char*)original - v, order, memory_order_relaxed));
+        return original;
+    }
+
+    bool
+    is_lock_free(void) const volatile BOOST_NOEXCEPT
+    {
+        return true;
+    }
+
+    BOOST_ATOMIC_DECLARE_VOID_POINTER_OPERATORS
+private:
+    base_atomic(const base_atomic &) /* = delete */ ;
+    void operator=(const base_atomic &) /* = delete */ ;
+    value_type v_;
+};
+
+template<typename T, bool Sign>
+class base_atomic<T *, void *, 8, Sign>
+{
+    typedef base_atomic this_type;
+    typedef T * value_type;
+    typedef ptrdiff_t difference_type;
+public:
+    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
+    base_atomic(void) {}
+
+    void
+    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        platform_fence_before_store(order);
+        platform_store64(v, &v_);
+        platform_fence_after_store(order);
+    }
+
+    value_type
+    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
+    {
+        value_type v = platform_load64(&v_);
+        platform_fence_after_load(order);
+        return v;
+    }
+
+    value_type
+    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        value_type original = load(memory_order_relaxed);
+        do {
+        } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
+        return original;
+    }
+
+    bool
+    compare_exchange_weak(
+        value_type & expected,
+        value_type desired,
+        memory_order success_order,
+        memory_order failure_order) volatile BOOST_NOEXCEPT
+    {
+        return compare_exchange_strong(expected, desired, success_order, failure_order);
+    }
+
+    bool
+    compare_exchange_strong(
+        value_type & expected,
+        value_type desired,
+        memory_order success_order,
+        memory_order failure_order) volatile BOOST_NOEXCEPT
+    {
+        platform_fence_before(success_order);
+
+        bool success = platform_cmpxchg64_strong(expected, desired, &v_);
+
+        if (success) {
+            platform_fence_after(success_order);
+        } else {
+            platform_fence_after(failure_order);
+        }
+
+        return success;
+    }
+
+    value_type
+    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        value_type original = load(memory_order_relaxed);
+        do {
+        } while (!compare_exchange_weak(original, original + v, order, memory_order_relaxed));
+        return original;
+    }
+
+    value_type
+    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        value_type original = load(memory_order_relaxed);
+        do {
+        } while (!compare_exchange_weak(original, original - v, order, memory_order_relaxed));
+        return original;
+    }
+
+    bool
+    is_lock_free(void) const volatile BOOST_NOEXCEPT
+    {
+        return true;
+    }
+
+    BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
+private:
+    base_atomic(const base_atomic &) /* = delete */ ;
+    void operator=(const base_atomic &) /* = delete */ ;
+    value_type v_;
+};
+
+}
+}
+}
+
+#endif

Modified: trunk/boost/atomic/detail/cas64strong.hpp
==============================================================================
--- trunk/boost/atomic/detail/cas64strong.hpp	Sat Jun  8 13:24:33 2013	(r84694)
+++ trunk/boost/atomic/detail/cas64strong.hpp	2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)	(r84695)
@@ -8,9 +8,9 @@
 //  Copyright (c) 2011 Helge Bahmann
 //  Copyright (c) 2013 Tim Blechmann
 
-// Build 64-bit atomic operation from platform_cmpxchg64_strong
+// Build 64-bit atomic operation on integers/UDTs from platform_cmpxchg64_strong
 // primitive. It is assumed that 64-bit loads/stores are not
-// atomic, so they are funnelled through cmpxchg as well.
+// atomic, so they are implemented through platform_load64/platform_store64.
 
 #include <cstddef>
 #include <boost/cstdint.hpp>
@@ -151,200 +151,6 @@
     value_type v_;
 };
 
-/* pointer types */
-
-template<bool Sign>
-class base_atomic<void *, void *, 8, Sign>
-{
-    typedef base_atomic this_type;
-    typedef void * value_type;
-    typedef ptrdiff_t difference_type;
-public:
-    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
-    base_atomic(void) {}
-
-    void
-    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        platform_fence_before_store(order);
-        platform_store64(v, &v_);
-        platform_fence_after_store(order);
-    }
-
-    value_type
-    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
-    {
-        value_type v = platform_load64(&v_);
-        platform_fence_after_load(order);
-        return v;
-    }
-
-    value_type
-    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        value_type original = load(memory_order_relaxed);
-        do {
-        } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
-        return original;
-    }
-
-    bool
-    compare_exchange_weak(
-        value_type & expected,
-        value_type desired,
-        memory_order success_order,
-        memory_order failure_order) volatile BOOST_NOEXCEPT
-    {
-        return compare_exchange_strong(expected, desired, success_order, failure_order);
-    }
-
-    bool
-    compare_exchange_strong(
-        value_type & expected,
-        value_type desired,
-        memory_order success_order,
-        memory_order failure_order) volatile BOOST_NOEXCEPT
-    {
-        platform_fence_before(success_order);
-
-        bool success = platform_cmpxchg64_strong(expected, desired, &v_);
-
-        if (success) {
-            platform_fence_after(success_order);
-        } else {
-            platform_fence_after(failure_order);
-        }
-
-        return success;
-    }
-
-    value_type
-    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        value_type original = load(memory_order_relaxed);
-        do {
-        } while (!compare_exchange_weak(original, (char*)original + v, order, memory_order_relaxed));
-        return original;
-    }
-
-    value_type
-    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        value_type original = load(memory_order_relaxed);
-        do {
-        } while (!compare_exchange_weak(original, (char*)original - v, order, memory_order_relaxed));
-        return original;
-    }
-
-    bool
-    is_lock_free(void) const volatile BOOST_NOEXCEPT
-    {
-        return true;
-    }
-
-    BOOST_ATOMIC_DECLARE_VOID_POINTER_OPERATORS
-private:
-    base_atomic(const base_atomic &) /* = delete */ ;
-    void operator=(const base_atomic &) /* = delete */ ;
-    value_type v_;
-};
-
-template<typename T, bool Sign>
-class base_atomic<T *, void *, 8, Sign>
-{
-    typedef base_atomic this_type;
-    typedef T * value_type;
-    typedef ptrdiff_t difference_type;
-public:
-    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT : v_(v) {}
-    base_atomic(void) {}
-
-    void
-    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        platform_fence_before_store(order);
-        platform_store64(v, &v_);
-        platform_fence_after_store(order);
-    }
-
-    value_type
-    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
-    {
-        value_type v = platform_load64(&v_);
-        platform_fence_after_load(order);
-        return v;
-    }
-
-    value_type
-    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        value_type original = load(memory_order_relaxed);
-        do {
-        } while (!compare_exchange_weak(original, v, order, memory_order_relaxed));
-        return original;
-    }
-
-    bool
-    compare_exchange_weak(
-        value_type & expected,
-        value_type desired,
-        memory_order success_order,
-        memory_order failure_order) volatile BOOST_NOEXCEPT
-    {
-        return compare_exchange_strong(expected, desired, success_order, failure_order);
-    }
-
-    bool
-    compare_exchange_strong(
-        value_type & expected,
-        value_type desired,
-        memory_order success_order,
-        memory_order failure_order) volatile BOOST_NOEXCEPT
-    {
-        platform_fence_before(success_order);
-
-        bool success = platform_cmpxchg64_strong(expected, desired, &v_);
-
-        if (success) {
-            platform_fence_after(success_order);
-        } else {
-            platform_fence_after(failure_order);
-        }
-
-        return success;
-    }
-
-    value_type
-    fetch_add(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        value_type original = load(memory_order_relaxed);
-        do {
-        } while (!compare_exchange_weak(original, original + v, order, memory_order_relaxed));
-        return original;
-    }
-
-    value_type
-    fetch_sub(difference_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        value_type original = load(memory_order_relaxed);
-        do {
-        } while (!compare_exchange_weak(original, original - v, order, memory_order_relaxed));
-        return original;
-    }
-
-    bool
-    is_lock_free(void) const volatile BOOST_NOEXCEPT
-    {
-        return true;
-    }
-
-    BOOST_ATOMIC_DECLARE_POINTER_OPERATORS
-private:
-    base_atomic(const base_atomic &) /* = delete */ ;
-    void operator=(const base_atomic &) /* = delete */ ;
-    value_type v_;
-};
-
 /* generic types */
 
 template<typename T, bool Sign>

Modified: trunk/boost/atomic/detail/gcc-x86.hpp
==============================================================================
--- trunk/boost/atomic/detail/gcc-x86.hpp	Sat Jun  8 13:24:33 2013	(r84694)
+++ trunk/boost/atomic/detail/gcc-x86.hpp	2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)	(r84695)
@@ -1655,14 +1655,16 @@
 
     In theory, could push/pop ebx onto/off the stack, but movs
     to a prepared stack slot turn out to be faster. */
-    __asm__ __volatile__ (
+    __asm__ __volatile__
+    (
         "movl %%ebx, %1\n"
         "movl %2, %%ebx\n"
         "lock; cmpxchg8b 0(%4)\n"
         "movl %1, %%ebx\n"
         : "=A" (prev), "=m" (scratch)
         : "D" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32)), "S" (ptr), "0" (prev)
-        : "memory");
+        : "memory"
+    );
     bool success = (prev == expected);
     expected = prev;
     return success;
@@ -1686,11 +1688,11 @@
 #if defined(__SSE2__)
         __asm__ __volatile__
         (
-            "movq %1, %%xmm0\n\t"
-            "movq %%xmm0, %0\n\t"
+            "movq %1, %%xmm4\n\t"
+            "movq %%xmm4, %0\n\t"
             : "=m" (*ptr)
             : "m" (value)
-            : "memory", "xmm0"
+            : "memory", "xmm4"
         );
 #else
         __asm__ __volatile__
@@ -1717,18 +1719,18 @@
 inline T
 platform_load64(const volatile T * ptr) BOOST_NOEXCEPT
 {
-    T value = T();
+    T value;
 
     if (((uint32_t)ptr & 0x00000007) == 0)
     {
 #if defined(__SSE2__)
         __asm__ __volatile__
         (
-            "movq %1, %%xmm0\n\t"
-            "movq %%xmm0, %0\n\t"
+            "movq %1, %%xmm4\n\t"
+            "movq %%xmm4, %0\n\t"
             : "=m" (value)
             : "m" (*ptr)
-            : "memory", "xmm0"
+            : "memory", "xmm4"
         );
 #else
         __asm__ __volatile__
@@ -1743,6 +1745,7 @@
     }
     else
     {
+        value = T();
         // We don't care for comparison result here; the previous value will be stored into value anyway.
         platform_cmpxchg64_strong(value, value, const_cast<volatile T*>(ptr));
     }

Modified: trunk/boost/atomic/detail/interlocked.hpp
==============================================================================
--- trunk/boost/atomic/detail/interlocked.hpp	Sat Jun  8 13:24:33 2013	(r84694)
+++ trunk/boost/atomic/detail/interlocked.hpp	2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)	(r84695)
@@ -43,6 +43,11 @@
 #define BOOST_ATOMIC_INTERLOCKED_OR(dest, arg) _InterlockedOr((long*)(dest), (long)(arg))
 #define BOOST_ATOMIC_INTERLOCKED_XOR(dest, arg) _InterlockedXor((long*)(dest), (long)(arg))
 
+#if (defined(_M_IX86) && _M_IX86 >= 500) || defined(_M_AMD64) || defined(_M_IA64)
+#pragma intrinsic(_InterlockedCompareExchange64)
+#define BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(dest, exchange, compare) _InterlockedCompareExchange64((__int64*)(dest), (__int64)(exchange), (__int64)(compare))
+#endif
+
 #if _MSC_VER >= 1600
 
 // MSVC 2010 and later provide intrinsics for 8 and 16 bit integers.
@@ -81,14 +86,12 @@
 
 #if defined(_M_AMD64) || defined(_M_IA64)
 
-#pragma intrinsic(_InterlockedCompareExchange64)
 #pragma intrinsic(_InterlockedExchangeAdd64)
 #pragma intrinsic(_InterlockedExchange64)
 #pragma intrinsic(_InterlockedAnd64)
 #pragma intrinsic(_InterlockedOr64)
 #pragma intrinsic(_InterlockedXor64)
 
-#define BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(dest, exchange, compare) _InterlockedCompareExchange64((__int64*)(dest), (__int64)(exchange), (__int64)(compare))
 #define BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(dest, addend) _InterlockedExchangeAdd64((__int64*)(dest), (__int64)(addend))
 #define BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(dest, newval) _InterlockedExchange64((__int64*)(dest), (__int64)(newval))
 #define BOOST_ATOMIC_INTERLOCKED_AND64(dest, arg) _InterlockedAnd64((__int64*)(dest), (__int64)(arg))

Modified: trunk/boost/atomic/detail/windows.hpp
==============================================================================
--- trunk/boost/atomic/detail/windows.hpp	Sat Jun  8 13:24:33 2013	(r84694)
+++ trunk/boost/atomic/detail/windows.hpp	2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)	(r84695)
@@ -179,7 +179,7 @@
 #define BOOST_ATOMIC_SHORT_LOCK_FREE 2
 #define BOOST_ATOMIC_INT_LOCK_FREE 2
 #define BOOST_ATOMIC_LONG_LOCK_FREE 2
-#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+#if (defined(_M_IX86) && _M_IX86 >= 500) || defined(_M_AMD64) || defined(_M_IA64)
 #define BOOST_ATOMIC_LLONG_LOCK_FREE 2
 #else
 #define BOOST_ATOMIC_LLONG_LOCK_FREE 0
@@ -716,161 +716,6 @@
     storage_type v_;
 };
 
-#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
-
-template<typename T, bool Sign>
-class base_atomic<T, int, 8, Sign>
-{
-    typedef base_atomic this_type;
-    typedef T value_type;
-    typedef value_type storage_type;
-    typedef T difference_type;
-public:
-    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT: v_(v) {}
-    base_atomic(void) {}
-
-    void
-    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        if (order != memory_order_seq_cst) {
-            platform_fence_before(order);
-            v_ = static_cast< storage_type >(v);
-        } else {
-            exchange(v, order);
-        }
-    }
-
-    value_type
-    load(memory_order order = memory_order_seq_cst)const volatile BOOST_NOEXCEPT
-    {
-        value_type v = static_cast< value_type >(v_);
-        platform_fence_after_load(order);
-        return v;
-    }
-
-    value_type
-    fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        platform_fence_before(order);
-        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&v_, v));
-        platform_fence_after(order);
-        return v;
-    }
-
-    value_type
-    fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        typedef typename make_signed< value_type >::type signed_value_type;
-        return fetch_add(static_cast< value_type >(-static_cast< signed_value_type >(v)), order);
-    }
-
-    value_type
-    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-        platform_fence_before(order);
-        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&v_, v));
-        platform_fence_after(order);
-        return v;
-    }
-
-    bool
-    compare_exchange_strong(
-        value_type & expected,
-        value_type desired,
-        memory_order success_order,
-        memory_order failure_order) volatile BOOST_NOEXCEPT
-    {
-        value_type previous = expected;
-        platform_fence_before(success_order);
-        value_type oldval = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&v_, desired, previous));
-        bool success = (previous == oldval);
-        if (success)
-            platform_fence_after(success_order);
-        else
-            platform_fence_after(failure_order);
-        expected = oldval;
-        return success;
-    }
-
-    bool
-    compare_exchange_weak(
-        value_type & expected,
-        value_type desired,
-        memory_order success_order,
-        memory_order failure_order) volatile BOOST_NOEXCEPT
-    {
-        return compare_exchange_strong(expected, desired, success_order, failure_order);
-    }
-
-    value_type
-    fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-#if defined(BOOST_ATOMIC_INTERLOCKED_AND64)
-        platform_fence_before(order);
-        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&v_, v));
-        platform_fence_after(order);
-        return v;
-#else
-        value_type tmp = load(memory_order_relaxed);
-        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
-        {
-            BOOST_ATOMIC_X86_PAUSE();
-        }
-        return tmp;
-#endif
-    }
-
-    value_type
-    fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-#if defined(BOOST_ATOMIC_INTERLOCKED_OR64)
-        platform_fence_before(order);
-        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&v_, v));
-        platform_fence_after(order);
-        return v;
-#else
-        value_type tmp = load(memory_order_relaxed);
-        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
-        {
-            BOOST_ATOMIC_X86_PAUSE();
-        }
-        return tmp;
-#endif
-    }
-
-    value_type
-    fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
-    {
-#if defined(BOOST_ATOMIC_INTERLOCKED_XOR64)
-        platform_fence_before(order);
-        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&v_, v));
-        platform_fence_after(order);
-        return v;
-#else
-        value_type tmp = load(memory_order_relaxed);
-        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
-        {
-            BOOST_ATOMIC_X86_PAUSE();
-        }
-        return tmp;
-#endif
-    }
-
-    bool
-    is_lock_free(void)const volatile BOOST_NOEXCEPT
-    {
-        return true;
-    }
-
-    BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
-private:
-    base_atomic(const base_atomic &) /* = delete */ ;
-    void operator=(const base_atomic &) /* = delete */ ;
-    storage_type v_;
-};
-
-#endif // defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
-
 // MSVC 2012 fails to recognize sizeof(T) as a constant expression in template specializations
 enum msvc_sizeof_pointer_workaround { sizeof_pointer = sizeof(void*) };
 
@@ -1362,7 +1207,7 @@
     }
 
     bool
-    is_lock_free(void)const volatile BOOST_NOEXCEPT
+    is_lock_free(void) const volatile BOOST_NOEXCEPT
     {
         return true;
     }
@@ -1374,7 +1219,158 @@
     storage_type v_;
 };
 
-#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+#if defined(_M_AMD64) || defined(_M_IA64)
+
+template<typename T, bool Sign>
+class base_atomic<T, int, 8, Sign>
+{
+    typedef base_atomic this_type;
+    typedef T value_type;
+    typedef value_type storage_type;
+    typedef T difference_type;
+public:
+    BOOST_CONSTEXPR explicit base_atomic(value_type v) BOOST_NOEXCEPT: v_(v) {}
+    base_atomic(void) {}
+
+    void
+    store(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        if (order != memory_order_seq_cst) {
+            platform_fence_before(order);
+            v_ = static_cast< storage_type >(v);
+        } else {
+            exchange(v, order);
+        }
+    }
+
+    value_type
+    load(memory_order order = memory_order_seq_cst) const volatile BOOST_NOEXCEPT
+    {
+        value_type v = static_cast< value_type >(v_);
+        platform_fence_after_load(order);
+        return v;
+    }
+
+    value_type
+    fetch_add(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        platform_fence_before(order);
+        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&v_, v));
+        platform_fence_after(order);
+        return v;
+    }
+
+    value_type
+    fetch_sub(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        typedef typename make_signed< value_type >::type signed_value_type;
+        return fetch_add(static_cast< value_type >(-static_cast< signed_value_type >(v)), order);
+    }
+
+    value_type
+    exchange(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+        platform_fence_before(order);
+        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&v_, v));
+        platform_fence_after(order);
+        return v;
+    }
+
+    bool
+    compare_exchange_strong(
+        value_type & expected,
+        value_type desired,
+        memory_order success_order,
+        memory_order failure_order) volatile BOOST_NOEXCEPT
+    {
+        value_type previous = expected;
+        platform_fence_before(success_order);
+        value_type oldval = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&v_, desired, previous));
+        bool success = (previous == oldval);
+        if (success)
+            platform_fence_after(success_order);
+        else
+            platform_fence_after(failure_order);
+        expected = oldval;
+        return success;
+    }
+
+    bool
+    compare_exchange_weak(
+        value_type & expected,
+        value_type desired,
+        memory_order success_order,
+        memory_order failure_order) volatile BOOST_NOEXCEPT
+    {
+        return compare_exchange_strong(expected, desired, success_order, failure_order);
+    }
+
+    value_type
+    fetch_and(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+#if defined(BOOST_ATOMIC_INTERLOCKED_AND64)
+        platform_fence_before(order);
+        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&v_, v));
+        platform_fence_after(order);
+        return v;
+#else
+        value_type tmp = load(memory_order_relaxed);
+        for (; !compare_exchange_weak(tmp, tmp & v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
+        return tmp;
+#endif
+    }
+
+    value_type
+    fetch_or(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+#if defined(BOOST_ATOMIC_INTERLOCKED_OR64)
+        platform_fence_before(order);
+        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&v_, v));
+        platform_fence_after(order);
+        return v;
+#else
+        value_type tmp = load(memory_order_relaxed);
+        for (; !compare_exchange_weak(tmp, tmp | v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
+        return tmp;
+#endif
+    }
+
+    value_type
+    fetch_xor(value_type v, memory_order order = memory_order_seq_cst) volatile BOOST_NOEXCEPT
+    {
+#if defined(BOOST_ATOMIC_INTERLOCKED_XOR64)
+        platform_fence_before(order);
+        v = static_cast< value_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&v_, v));
+        platform_fence_after(order);
+        return v;
+#else
+        value_type tmp = load(memory_order_relaxed);
+        for (; !compare_exchange_weak(tmp, tmp ^ v, order, memory_order_relaxed);)
+        {
+            BOOST_ATOMIC_X86_PAUSE();
+        }
+        return tmp;
+#endif
+    }
+
+    bool
+    is_lock_free(void)const volatile BOOST_NOEXCEPT
+    {
+        return true;
+    }
+
+    BOOST_ATOMIC_DECLARE_INTEGRAL_OPERATORS
+private:
+    base_atomic(const base_atomic &) /* = delete */ ;
+    void operator=(const base_atomic &) /* = delete */ ;
+    storage_type v_;
+};
 
 template<typename T, bool Sign>
 class base_atomic<T, void, 8, Sign>
@@ -1469,12 +1465,145 @@
     storage_type v_;
 };
 
-#endif // defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+#elif defined(_M_IX86) && _M_IX86 >= 500
+
+template<typename T>
+inline bool
+platform_cmpxchg64_strong(T & expected, T desired, volatile T * p) BOOST_NOEXCEPT
+{
+#if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
+    const T oldval = BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(p, desired, expected);
+    const bool result = (oldval == expected);
+    expected = oldval;
+    return result;
+#else
+    bool result;
+    __asm
+    {
+        mov edi, p
+        mov esi, expected
+        mov ebx, dword ptr [desired]
+        mov ecx, dword ptr [desired + 4]
+        mov eax, dword ptr [esi]
+        mov edx, dword ptr [esi + 4]
+        lock cmpxchg8b qword ptr [edi]
+        mov dword ptr [esi], eax
+        mov dword ptr [esi + 4], edx
+#if _M_IX86 >= 600
+        sete result
+#else
+        mov al, 0
+        jne not_equal_label
+        mov al, 1
+not_equal_label:
+        mov result, al
+#endif
+    };
+    return result;
+#endif
+}
+
+// Intel 64 and IA-32 Architectures Software Developer's Manual, Volume 3A, 8.1.1. Guaranteed Atomic Operations:
+//
+// The Pentium processor (and newer processors since) guarantees that the following additional memory operations will always be carried out atomically:
+// * Reading or writing a quadword aligned on a 64-bit boundary
+//
+// Luckily, the memory is almost always 8-byte aligned in our case because atomic<> uses 64 bit native types for storage and dynamic memory allocations
+// have at least 8 byte alignment. The only unfortunate case is when atomic is placeod on the stack and it is not 8-byte aligned (like on 32 bit Windows).
+
+template<typename T>
+inline void
+platform_store64(T value, volatile T * p) BOOST_NOEXCEPT
+{
+    if (((uint32_t)p & 0x00000007) == 0)
+    {
+#if defined(_M_IX86_FP) && _M_IX86_FP >= 2
+        __asm
+        {
+            mov edx, p
+            movq xmm4, value
+            movq qword ptr [edx], xmm4
+        };
+#else
+        __asm
+        {
+            mov edx, p
+            fild value
+            fistp qword ptr [edx]
+        };
+#endif
+    }
+    else
+    {
+        __asm
+        {
+            mov edi, p
+            mov ebx, dword ptr [value]
+            mov ecx, dword ptr [value + 4]
+            mov eax, dword ptr [edi]
+            mov edx, dword ptr [edi + 4]
+            align 16
+again:
+            lock cmpxchg8b qword ptr [edi]
+            jne again
+        };
+    }
+}
+
+template<typename T>
+inline T
+platform_load64(const volatile T * p) BOOST_NOEXCEPT
+{
+    T value;
+
+    if (((uint32_t)p & 0x00000007) == 0)
+    {
+#if defined(_M_IX86_FP) && _M_IX86_FP >= 2
+        __asm
+        {
+            mov edx, p
+            movq xmm4, qword ptr [edx]
+            movq value, xmm4
+        };
+#else
+        __asm
+        {
+            mov edx, p
+            fild qword ptr [edx]
+            fistp value
+        };
+#endif
+    }
+    else
+    {
+        // We don't care for comparison result here; the previous value will be stored into value anyway.
+        __asm
+        {
+            mov edi, p
+            xor ebx, ebx
+            xor ecx, ecx
+            xor eax, eax
+            xor edx, edx
+            lock cmpxchg8b qword ptr [edi]
+            mov dword ptr [value], eax
+            mov dword ptr [value + 4], edx
+        };
+    }
+
+    return value;
+}
+
+#endif
 
 } // namespace detail
 } // namespace atomics
 } // namespace boost
 
+/* pull in 64-bit atomic type using cmpxchg8b above */
+#if defined(_M_IX86) && _M_IX86 >= 500
+#include <boost/atomic/detail/cas64strong.hpp>
+#endif
+
 #endif /* !defined(BOOST_ATOMIC_FORCE_FALLBACK) */
 
 #ifdef _MSC_VER

Modified: trunk/libs/atomic/test/lockfree.cpp
==============================================================================
--- trunk/libs/atomic/test/lockfree.cpp	Sat Jun  8 13:24:33 2013	(r84694)
+++ trunk/libs/atomic/test/lockfree.cpp	2013-06-08 13:36:38 EDT (Sat, 08 Jun 2013)	(r84695)
@@ -129,7 +129,7 @@
 #define EXPECT_SHORT_LOCK_FREE 2
 #define EXPECT_INT_LOCK_FREE 2
 #define EXPECT_LONG_LOCK_FREE 2
-#if defined(_WIN64)
+#if defined(_WIN64) || (defined(_M_IX86) && _M_IX86 >= 500) || defined(_M_AMD64) || defined(_M_IA64)
 #define EXPECT_LLONG_LOCK_FREE 2
 #else
 #define EXPECT_LLONG_LOCK_FREE 0