[Boost-commit] svn:boost r55341 - in sandbox: boost/mapreduce boost/mapreduce/intermediates libs/mapreduce/test

Subject: [Boost-commit] svn:boost r55341 - in sandbox: boost/mapreduce boost/mapreduce/intermediates libs/mapreduce/test
From: cdm.henderson_at_[hidden]
Date: 2009-08-01 09:41:24

Author: chenderson
Date: 2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
New Revision: 55341
URL: http://svn.boost.org/trac/boost/changeset/55341

Log:
Small optimisation for Combining results at the end of the Map Task
Text files modified: 
   sandbox/boost/mapreduce/intermediates/in_memory.hpp |     5 +----                                   
   sandbox/boost/mapreduce/job.hpp                     |     3 +--                                     
   sandbox/boost/mapreduce/null_combiner.hpp           |     5 +++++                                   
   sandbox/libs/mapreduce/test/mrtest.cpp              |    18 +++++++++++++++---                      
   4 files changed, 22 insertions(+), 9 deletions(-)

Modified: sandbox/boost/mapreduce/intermediates/in_memory.hpp
==============================================================================
--- sandbox/boost/mapreduce/intermediates/in_memory.hpp	(original)
+++ sandbox/boost/mapreduce/intermediates/in_memory.hpp	2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -125,16 +125,13 @@
             {
                 fn_obj.start(it1->first);
                 std::for_each<
-                    intermediates_t::value_type::mapped_type::const_iterator, FnObj &>(
+                    typename intermediates_t::value_type::mapped_type::const_iterator, FnObj &>(
                         it1->second.begin(),
                         it1->second.end(),
                         fn_obj);
                 fn_obj.finish(it1->first, *this);
             }
         }
-//!!!null_combiner
-        //if (intermediates_.size() == 0)
-        //    std::swap(intermediates_, intermediates);
     }
 
   private:

Modified: sandbox/boost/mapreduce/job.hpp
==============================================================================
--- sandbox/boost/mapreduce/job.hpp	(original)
+++ sandbox/boost/mapreduce/job.hpp	2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -73,8 +73,7 @@
 
             // consolidating map intermediate results can save network time by
             // aggregating the mapped valued at mapper
-            combiner_type combiner;
-            intermediate_store_.combine(combiner);
+            combiner_type::run(intermediate_store_);
 
             return *this;
         }

Modified: sandbox/boost/mapreduce/null_combiner.hpp
==============================================================================
--- sandbox/boost/mapreduce/null_combiner.hpp	(original)
+++ sandbox/boost/mapreduce/null_combiner.hpp	2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -19,6 +19,11 @@
 
 struct null_combiner
 {
+    template<typename IntermediateStore>
+    static void run(IntermediateStore &/*intermediate_store*/)
+    {
+    }
+
     template<typename IntermediateValueType>
     void start(IntermediateValueType const &)
     { }

Modified: sandbox/libs/mapreduce/test/mrtest.cpp
==============================================================================
--- sandbox/libs/mapreduce/test/mrtest.cpp	(original)
+++ sandbox/libs/mapreduce/test/mrtest.cpp	2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -16,7 +16,9 @@
 #define USE_IN_MEMORY_INTERMEDIATES
 
 #if defined(_DEBUG)
-#   define RUN_SEQUENTIAL_MAP_REDUCE
+#   if 0
+#       define RUN_SEQUENTIAL_MAP_REDUCE
+#   endif
 #else
 #   define BOOST_DISABLE_ASSERTS
 #endif
@@ -176,9 +178,9 @@
 #endif
   , boost::mapreduce::datasource::directory_iterator<wordcount::map_task_type>
 #ifdef USE_IN_MEMORY_INTERMEDIATES
-  , boost::mapreduce::intermediates::local_disk<wordcount::map_task_type, wordcount::reduce_task>
-#else
   , boost::mapreduce::intermediates::in_memory<wordcount::map_task_type, wordcount::reduce_task>
+#else
+  , boost::mapreduce::intermediates::local_disk<wordcount::map_task_type, wordcount::reduce_task>
 #endif
   , boost::mapreduce::intermediates::reduce_file_output<wordcount::map_task_type, wordcount::reduce_task>
 > job;
@@ -188,6 +190,13 @@
 class combiner
 {
   public:
+    template<typename IntermediateStore>
+    static void run(IntermediateStore &intermediate_store)
+    {
+        combiner instance;
+        intermediate_store.combine(instance);
+    }
+
     void start(job::map_task_type::intermediate_key_type const &)
     {
         total_ = 0;
@@ -206,6 +215,9 @@
     }
 
   private:
+    combiner() { }
+
+  private:
     unsigned total_;
 };