[boost] memory corruption in async_read_until using asio::ssl

Subject: [boost] memory corruption in async_read_until using asio::ssl
From: Jim Page (jim.page_at_[hidden])
Date: 2012-08-27 11:04:13

Afternoon All

I am seeing an occasional memory corruption bug, using valgrind, in our web
server application that is using boost::asio::ssl, and it seems to be
related to calls to async_read_until, where several chunks of data are
required before the terminating regex is found.

What appears to be happening is that engine::perform() causes a write (via
ssl3_read_bytes()) on a deleted memory region, the buffer into which
decrypted data is written. This region was delete'd via invocation of the
detail::io_op::handler, and in turn via read_until_delim_op::operator(), due
to prepare() being called on the streambuf, and a std::vector<char> is
delete'd in order to create a larger one.

It seems like one of the chain of handlers is using a pointer which is
becomes invalid when the streambuff is expanded after failing to find the
regex terminator after one or more reads, and prepare() is called before the
next async_read_some() on the underlying stream.

I have included the valgrind output at the end - sorry for the large size.

I am pretty certain that I have ruled out the obvious (multiple simultaneous
attempts to read using the same streambuf), and I am only using one thread
for the io_service.

This seems a pretty common potential failure mode, so perhaps I am barking
up the wrong tree - I'd appreciate anyone's opinion before I start a
potentially tricky debug effort requiring modification to the boost
libraries on our production servers.

I'm using boost 1.49. There's a lot of optimization on the application which
is why some calls are optimized out of the valgrind dump. The machines are
running Centos 5.8, x86_64. Compiler is gcc 4.4.6.

Thanks for looking
Jim Page

==29870== Thread 22:
==29870== Invalid write of size 1
==29870==    at 0x4A088F8: memcpy (mc_replace_strmem.c:587)
==29870==    by 0x3400821EDE: ssl3_read_bytes (in /lib64/libssl.so.0.9.8e)
==29870==    by 0x340081E750: ??? (in /lib64/libssl.so.0.9.8e)
==29870==    by 0x66A75B: boost::asio::ssl::detail::engine::perform(int
(boost::asio::ssl::detail::engine::*)(void*, unsigned long), void*, unsigned
long, boost::system::error_code&, unsigned long*) (engine.ipp:219)
==29870==    by 0x7E3326:
boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio
::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >,
boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>,
boost::asio::detail::rea
d_until_expr_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<bo
ost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp>
> >, std::allocator<char>, boost::basic_regex<char,
boost::regex_traits<char, boost::cpp_regex_traits<char> > >,
boost::_bi::bind_t<void, boost::_mfi::mf2<void, HttpConnection,
boost::system::error_code const&, char const*>,
boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >,
boost::arg<1> (*)(), boost::_bi::value<char const*> > > >
>::operator()(boost::system::error_code, unsigned long, int)
(engine.ipp:151)
==29870==    by 0x7E45C2:
boost::asio::detail::wait_handler<boost::asio::ssl::detail::io_op<boost::asi
o::basic_stream_socket<boost::asio::ip::tcp,
boost::asio::stream_socket_service<boost::asio::ip::tcp> >,
boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>,
boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio
::basic_stream_socket<boost::asio::ip::tcp,
boost::asio::stream_socket_service<boost::asio::ip::tcp> > >,
std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char,
boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void,
boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&,
char const*>,
boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >,
boost::arg<1> (*)(), boost::_bi::value<char const*> > > > >
>::do_complete(boost::asio::detail::task_io_service*,
boost::asio::detail::task_io_service_operation*, boost::system::error_code
const&, unsigned long) (bind_handler.hpp:46)
==29870==    by 0x619783:
boost::asio::detail::task_io_service::run(boost::system::error_code&)
(task_io_service_operation.hpp:37)
==29870==    by 0x79F4FE: BayeuxServer::run() (io_service.ipp:59)
==29870==    by 0x364800DB8E: ??? (in /usr/lib64/libboost_thread.so.1.49.0)
==29870==    by 0x3AE300677C: start_thread (in /lib64/libpthread-2.5.so)
==29870==    by 0x3AE20D325C: clone (in /lib64/libc-2.5.so)
==29870==  Address 0x1b508610 is 0 bytes inside a block of size 1,024 free'd
==29870==    at 0x4A05A33: operator delete(void*) (vg_replace_malloc.c:346)
==29870==    by 0x658E54: std::vector<char, std::allocator<char>
>::_M_fill_insert(__gnu_cxx::__normal_iterator<char*, std::vector<char,
std::allocator<char> > >, unsigned long, char const&) (new_allocator.h:95)
==29870==    by 0x68B805: boost::asio::basic_streambuf<std::allocator<char>
>::reserve(unsigned long) (stl_vector.h:851)
==29870==    by 0x7E2EF2:
boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio
::basic_stream_socket<boost::asio::ip::tcp,
boost::asio::stream_socket_service<boost::asio::ip::tcp> > >,
std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char,
boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void,
boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&,
char const*>,
boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >,
boost::arg<1> (*)(), boost::_bi::value<char const*> > >
>::operator()(boost::system::error_code const&, unsigned long, int)
(basic_streambuf.hpp:207)
==29870==    by 0x7E32DE:
boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio
::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >,
boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>,
boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio
::basic_stream_socket<boost::asio::ip::tcp,
boost::asio::stream_socket_service<boost::asio::ip::tcp> > >,
std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char,
boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void,
boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&,
char const*>,
boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >,
boost::arg<1> (*)(), boost::_bi::value<char const*> > > >
>::operator()(boost::system::error_code, unsigned long, int)
(read_op.hpp:59)
==29870==    by 0x7E493C:
boost::asio::detail::reactive_socket_recv_op<boost::asio::mutable_buffers_1,
boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio
::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >,
boost::asio::ssl::detail::read_op<boost::asio::mutable_buffers_1>,
boost::asio::detail::read_until_expr_op<boost::asio::ssl::stream<boost::asio
::basic_stream_socket<boost::asio::ip::tcp,
boost::asio::stream_socket_service<boost::asio::ip::tcp> > >,
std::allocator<char>, boost::basic_regex<char, boost::regex_traits<char,
boost::cpp_regex_traits<char> > >, boost::_bi::bind_t<void,
boost::_mfi::mf2<void, HttpConnection, boost::system::error_code const&,
char const*>,
boost::_bi::list3<boost::_bi::value<boost::shared_ptr<HttpConnection> >,
boost::arg<1> (*)(), boost::_bi::value<char const*> > > > >
>::do_complete(boost::asio::detail::task_io_service*,
boost::asio::detail::task_io_service_operation*, boost::system::error_code
const&, unsigned long) (bind_handler.hpp:118)
==29870==    by 0x613778:
boost::asio::detail::epoll_reactor::descriptor_state::do_complete(boost::asi
o::detail::task_io_service*,
boost::asio::detail::task_io_service_operation*, boost::system::error_code
const&, unsigned long) (task_io_service_operation.hpp:37)
==29870==    by 0x619783:
boost::asio::detail::task_io_service::run(boost::system::error_code&)
(task_io_service_operation.hpp:37)
==29870==    by 0x79F4FE: BayeuxServer::run() (io_service.ipp:59)
==29870==    by 0x364800DB8E: ??? (in /usr/lib64/libboost_thread.so.1.49.0)
==29870==    by 0x3AE300677C: start_thread (in /lib64/libpthread-2.5.so)
==29870==    by 0x3AE20D325C: clone (in /lib64/libc-2.5.so)