[Boost-users] [Interprocess] : problem with large vector of maps in memory mapped file

Subject: [Boost-users] [Interprocess] : problem with large vector of maps in memory mapped file
From: Steven Maenhout (Steven.Maenhout_at_[hidden])
Date: 2011-07-11 04:18:43

Dear all,

I'm using Boost 1.47 on Linux x86_64 in combination with gcc version 
4.5.1. I'm trying to implement a large vector of maps in a memory mapped 
file. My code works fine on small example cases but crashes randomly 
when trying to store larger datasets. The application generally runs 
fine for a few days before a segementation fault occurs which makes 
debugging very time consuming. A trimmed down version of the code looks 
like this:

  typedef boost::interprocess::managed_mapped_file::segment_manager 
SegmentManager_t;
typedef boost::interprocess::allocator<void, SegmentManager_t> 
VoidAllocator_t;
typedef std::pair<const uint,double> MapValue_t;
typedef boost::interprocess::allocator<MapValue_t , SegmentManager_t> 
MapValueAllocator_t;
typedef 
boost::interprocess::map<uint,double,std::less<uint>,MapValueAllocator_t> Map_t;
typedef boost::interprocess::allocator<Map_t,SegmentManager_t> 
MapAllocator_t;
typedef boost::interprocess::vector<Map_t,MapAllocator_t> VectorofMaps_t;

boost::interprocess::managed_mapped_file 
m_mfile=managed_mapped_file(open_or_create, TEMPFILEPATH, TEMPFILEPATHSIZE);
VoidAllocator_t alloc_inst (m_mfile.get_segment_manager());
VectorofMaps_t* 
m_pS=m_mfile.construct<VectorofMaps_t>(anonymous_instance)(numsparserows,Map_t(std::less<uint>(),alloc_inst),alloc_inst);

I've managed to obtain a stack trace which pinpoints the problem at the 
use of the function lower_bound:

Map_t::iterator mapiter=(*m_pS)[row].lower_bound(col);

I realize that such random segfaults could be the result of a hardware 
problem (memory corruption, cpu overheating, hard drive storing the 
memory mapped file (ssd)) but stress testing these components has not 
revealed any problems. I am therefore hoping that somebody would be able 
to get some more information from the stack trace provided below. Any 
help or advice on how to identify the problem would be greatly appreciated.

Steven

#0  0x0000000000720244 in std::less<unsigned int>::operator() 
(this=0x7fffffffc390, __x=@0x3ff07fcf9f552438, __y=@0x7fffffffc5c0)
     at 
/usr/lib/gcc/x86_64-redhat-linux/4.5.1/../../../../include/c++/4.5.1/bits/stl_function.h:230
#1  0x0000000000721fc7 in 
boost::container::containers_detail::value_compare_impl<unsigned int, 
std::pair<unsigned int const, double>, std::less<unsigned int>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> > >::operator()<std::pair<unsigned int const, double>, 
unsigned int> (this=0x7fffffffc390, key1=..., key2=@0x7fffffffc5c0)
     at 
/home/smaenhout/c++/boost/include/boost/interprocess/containers/container/detail/tree.hpp:78
#2  0x0000000000721b4f in 
boost::container::containers_detail::rbtree<unsigned int, 
std::pair<unsigned int const, double>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> >, std::less<unsigned int>, 
boost::interprocess::allocator<std::pair<unsigned int const, double>, 
boost::interprocess::segment_manager<char, 
boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, 
boost::interprocess::offset_ptr<void, long, unsigned long, 0u>, 0ul>, 
boost::interprocess::iset_index> > 
 >::key_node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > > >::operator()<boost::container::containers_detail::rbtree_node<std::pair<unsigned int const, double>, boost::interprocess::offset_ptr<void> >, unsigned int> (this=0x7fffffffc390, key1=..., key2=@0x7fffffffc5c0)
     at 
/home/smaenhout/c++/boost/include/boost/interprocess/containers/container/detail/tree.hpp:337
#3  0x0000000000721747 in 
boost::intrusive::detail::key_nodeptr_comp<boost::container::containers_detail::rbtree<unsigned 
int, std::pair<unsigned int const, double>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> >, std::less<unsigned int>, 
boost::interprocess::allocator<std::pair<unsigned int const, double>, 
boost::interprocess::segment_manager<char, 
boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family>, 
boost::interprocess::iset_index> > 
 >::key_node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > > >, boost::intrusive::rbtree_impl<boost::intrusive::setopt<boost::intrusive::detail::base_hook_traits<boost::container::containers_detail::rbtree_node<std::pair<unsigned int const, double>, boost::interprocess::offset_ptr<void, long, unsigned long, 0u> >, boost::intrusive::rbtree_node_traits<boost::interprocess::offset_ptr<void, long, unsigned long, 0u>, true>, (boost::intrusive::link_mode_type)0, boost::intrusive::default_tag, 3>, boost::container::containers_detail::node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > >, boost::container::containers_detail::
rbtree_node<std::pair<unsigned int const, double>, boost::interprocess::offset_ptr<void, long, unsigned long, 0u> > >, unsigned long, true> > >::operator()<boost::interprocess::offset_ptr<boost::intrusive::compact_rbtree_node<boost::interprocess::offset_ptr<void> >, long, unsigned long, 0u>, unsigned int> (this=0x7fffffffc390, key1=..., key2=@0x7fffffffc5c0)
     at 
/home/smaenhout/c++/boost/include/boost/intrusive/detail/utilities.hpp:236
#4  0x00000000007213cc in 
boost::intrusive::detail::tree_algorithms<boost::intrusive::rbtree_node_traits<boost::interprocess::offset_ptr<void, 
long, unsigned long, 0u>, true> >::lower_bound<unsigned int, 
boost::intrusive::detail::key_nodeptr_comp<boost::container::containers_detail::rbtree<unsigned 
int, std::pair<unsigned int const, double>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> >, std::less<unsigned int>, 
boost::interprocess::allocator<std::pair<unsigned int const, double>, 
boost::interprocess::segment_manager<char, 
boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family>, 
boost::interprocess::iset_index> > 
 >::key_node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > > >, boost::intrusive::rbtree_impl<boost::intrusive::setopt<boost::intrusive::detail::base_hook_traits<boost::container::containers_detail::rbtree_node<std::pair<unsigned int const, double>, boost::interprocess::offset_ptr<void> >, boost::intrusive::rbtree_node_traits<boost::interprocess::offset_ptr<void>, true>, (boost::intrusive::link_mode_type)0, boost::intrusive::default_tag, 3>, boost::container::containers_detail::node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > >, boost::container::containers_detail::rbtree_node<std::pair<unsigned int const, double>,
 boost::interprocess::offset_ptr<void> > >, unsigned long, true> > > > (header=..., key=@0x7fffffffc5c0, comp=...)
     at 
/home/smaenhout/c++/boost/include/boost/intrusive/detail/tree_algorithms.hpp:839
#5  0x0000000000720e32 in 
boost::intrusive::rbtree_algorithms<boost::intrusive::rbtree_node_traits<boost::interprocess::offset_ptr<void, 
long, unsigned long, 0u>, true> >::lower_bound<unsigned int, 
boost::intrusive::detail::key_nodeptr_comp<boost::container::containers_detail::rbtree<unsigned 
int, std::pair<unsigned int const, double>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> >, std::less<unsigned int>, 
boost::interprocess::allocator<std::pair<unsigned int const, double>, 
boost::interprocess::segment_manager<char, 
boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family>, 
boost::interprocess::iset_index> > 
 >::key_node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > > >, boost::intrusive::rbtree_impl<boost::intrusive::setopt<boost::intrusive::detail::base_hook_traits<boost::container::containers_detail::rbtree_node<std::pair<unsigned int const, double>, boost::interprocess::offset_ptr<void> >, boost::intrusive::rbtree_node_traits<boost::interprocess::offset_ptr<void>, true>, (boost::intrusive::link_mode_type)0, boost::intrusive::default_tag, 3>, boost::container::containers_detail::node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > >, boost::container::containers_detail::rbtree_node<std::pair<unsigned int const, double>,
 boost::interprocess::offset_ptr<void> > >, unsigned long, true> > > > (header=..., key=@0x7fffffffc5c0, comp=...) at /home/smaenhout/c++/boost/include/boost/intrusive/rbtree_algorithms.hpp:470
#6  0x00000000007207f1 in 
boost::intrusive::rbtree_impl<boost::intrusive::setopt<boost::intrusive::detail::base_hook_traits<boost::container::containers_detail::rbtree_node<std::pair<unsigned 
int const, double>, boost::interprocess::offset_ptr<void, long, unsigned 
long, 0u> >, 
boost::intrusive::rbtree_node_traits<boost::interprocess::offset_ptr<void, 
long, unsigned long, 0u>, true>, (boost::intrusive::link_mode_type)0, 
boost::intrusive::default_tag, 3>, 
boost::container::containers_detail::node_compare<boost::container::containers_detail::value_compare_impl<unsigned 
int, std::pair<unsigned int const, double>, std::less<unsigned int>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> > >, 
boost::container::containers_detail::rbtree_node<std::pair<unsigned int 
const, double>, boost::interprocess::offset_ptr<void, long, unsigned 
long, 0u> > >, unsigned long, true> >::lower_bound<unsigned int, 
boost::container::containers_detail::rbtree<unsigned int, 
std::pair<unsigned int const, double>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> >, std::less<unsigned int>, 
boost::interprocess::allocator<std::pair<unsigned int const, double>, 
boost::interprocess::segment_manager<char, 
boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family>, 
boost::interprocess::iset_index> > 
 >::key_node_compare<boost::container::containers_detail::value_compare_impl<unsigned int, std::pair<unsigned int const, double>, std::less<unsigned int>, boost::container::containers_detail::select1st<std::pair<unsigned int const, double> > > > > (this=0x7fc8f21bef08,
     key=@0x7fffffffc5c0, comp=...) at 
/home/smaenhout/c++/boost/include/boost/intrusive/rbtree.hpp:1064
#7  0x00000000007202ec in 
boost::container::containers_detail::rbtree<unsigned int, 
std::pair<unsigned int const, double>, 
boost::container::containers_detail::select1st<std::pair<unsigned int 
const, double> >, std::less<unsigned int>, 
boost::interprocess::allocator<std::pair<unsigned int const, double>, 
boost::interprocess::segment_manager<char, 
boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, 
boost::interprocess::offset_ptr<void, long, unsigned long, 0u>, 0ul>, 
boost::interprocess::iset_index> > >::lower_bound (this=0x7fc8f21bef00, 
k=@0x7fffffffc5c0)
     at 
/home/smaenhout/c++/boost/include/boost/interprocess/containers/container/detail/tree.hpp:853
#8  0x000000000071fea8 in boost::container::map<unsigned int, double, 
std::less<unsigned int>, 
boost::interprocess::allocator<std::pair<unsigned int const, double>, 
boost::interprocess::segment_manager<char, 
boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, 
boost::interprocess::offset_ptr<void, long, unsigned long, 0u>, 0ul>, 
boost::interprocess::iset_index> > >::lower_bound (
     this=0x7fc8f21bef00, x=@0x7fffffffc5c0) at 
/home/smaenhout/c++/boost/include/boost/interprocess/containers/container/map.hpp:638