Boost logo

Boost-Commit :

Subject: [Boost-commit] svn:boost r55341 - in sandbox: boost/mapreduce boost/mapreduce/intermediates libs/mapreduce/test
From: cdm.henderson_at_[hidden]
Date: 2009-08-01 09:41:24


Author: chenderson
Date: 2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
New Revision: 55341
URL: http://svn.boost.org/trac/boost/changeset/55341

Log:
Small optimisation for Combining results at the end of the Map Task
Text files modified:
   sandbox/boost/mapreduce/intermediates/in_memory.hpp | 5 +----
   sandbox/boost/mapreduce/job.hpp | 3 +--
   sandbox/boost/mapreduce/null_combiner.hpp | 5 +++++
   sandbox/libs/mapreduce/test/mrtest.cpp | 18 +++++++++++++++---
   4 files changed, 22 insertions(+), 9 deletions(-)

Modified: sandbox/boost/mapreduce/intermediates/in_memory.hpp
==============================================================================
--- sandbox/boost/mapreduce/intermediates/in_memory.hpp (original)
+++ sandbox/boost/mapreduce/intermediates/in_memory.hpp 2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -125,16 +125,13 @@
             {
                 fn_obj.start(it1->first);
                 std::for_each<
- intermediates_t::value_type::mapped_type::const_iterator, FnObj &>(
+ typename intermediates_t::value_type::mapped_type::const_iterator, FnObj &>(
                         it1->second.begin(),
                         it1->second.end(),
                         fn_obj);
                 fn_obj.finish(it1->first, *this);
             }
         }
-//!!!null_combiner
- //if (intermediates_.size() == 0)
- // std::swap(intermediates_, intermediates);
     }
 
   private:

Modified: sandbox/boost/mapreduce/job.hpp
==============================================================================
--- sandbox/boost/mapreduce/job.hpp (original)
+++ sandbox/boost/mapreduce/job.hpp 2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -73,8 +73,7 @@
 
             // consolidating map intermediate results can save network time by
             // aggregating the mapped valued at mapper
- combiner_type combiner;
- intermediate_store_.combine(combiner);
+ combiner_type::run(intermediate_store_);
 
             return *this;
         }

Modified: sandbox/boost/mapreduce/null_combiner.hpp
==============================================================================
--- sandbox/boost/mapreduce/null_combiner.hpp (original)
+++ sandbox/boost/mapreduce/null_combiner.hpp 2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -19,6 +19,11 @@
 
 struct null_combiner
 {
+ template<typename IntermediateStore>
+ static void run(IntermediateStore &/*intermediate_store*/)
+ {
+ }
+
     template<typename IntermediateValueType>
     void start(IntermediateValueType const &)
     { }

Modified: sandbox/libs/mapreduce/test/mrtest.cpp
==============================================================================
--- sandbox/libs/mapreduce/test/mrtest.cpp (original)
+++ sandbox/libs/mapreduce/test/mrtest.cpp 2009-08-01 09:41:23 EDT (Sat, 01 Aug 2009)
@@ -16,7 +16,9 @@
 #define USE_IN_MEMORY_INTERMEDIATES
 
 #if defined(_DEBUG)
-# define RUN_SEQUENTIAL_MAP_REDUCE
+# if 0
+# define RUN_SEQUENTIAL_MAP_REDUCE
+# endif
 #else
 # define BOOST_DISABLE_ASSERTS
 #endif
@@ -176,9 +178,9 @@
 #endif
   , boost::mapreduce::datasource::directory_iterator<wordcount::map_task_type>
 #ifdef USE_IN_MEMORY_INTERMEDIATES
- , boost::mapreduce::intermediates::local_disk<wordcount::map_task_type, wordcount::reduce_task>
-#else
   , boost::mapreduce::intermediates::in_memory<wordcount::map_task_type, wordcount::reduce_task>
+#else
+ , boost::mapreduce::intermediates::local_disk<wordcount::map_task_type, wordcount::reduce_task>
 #endif
   , boost::mapreduce::intermediates::reduce_file_output<wordcount::map_task_type, wordcount::reduce_task>
> job;
@@ -188,6 +190,13 @@
 class combiner
 {
   public:
+ template<typename IntermediateStore>
+ static void run(IntermediateStore &intermediate_store)
+ {
+ combiner instance;
+ intermediate_store.combine(instance);
+ }
+
     void start(job::map_task_type::intermediate_key_type const &)
     {
         total_ = 0;
@@ -206,6 +215,9 @@
     }
 
   private:
+ combiner() { }
+
+ private:
     unsigned total_;
 };
 


Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk