[Boost-commit] svn:boost r46001 - in trunk: boost/iostreams/filter libs/iostreams/test

From: technews_at_[hidden]
Date: 2008-05-31 18:53:58

Author: turkanis
Date: 2008-05-31 18:53:58 EDT (Sat, 31 May 2008)
New Revision: 46001
URL: http://svn.boost.org/trac/boost/changeset/46001

Log:
added support for archives with multiple members; added tests for metadata and for multiple members (fixes #1896)
Text files modified: 
   trunk/boost/iostreams/filter/gzip.hpp   |   516 +++++++++++++++++++++++++++++---------- 
   trunk/libs/iostreams/test/gzip_test.cpp |    65 ++++                                    
   2 files changed, 432 insertions(+), 149 deletions(-)

Modified: trunk/boost/iostreams/filter/gzip.hpp
==============================================================================
--- trunk/boost/iostreams/filter/gzip.hpp	(original)
+++ trunk/boost/iostreams/filter/gzip.hpp	2008-05-31 18:53:58 EDT (Sat, 31 May 2008)
@@ -19,6 +19,7 @@
 #include <boost/config.hpp> // STATIC_CONSTANT, STDC_NAMESPACE, 
                             // DINKUMWARE_STDLIB, __STL_CONFIG_H.
 #include <algorithm>                      // min.
+#include <cassert>
 #include <cstdio>                         // EOF.
 #include <cstddef>                        // size_t.
 #include <ctime>                          // std::time_t.
@@ -34,7 +35,8 @@
 #include <boost/iostreams/operations.hpp>
 #include <boost/iostreams/device/back_inserter.hpp>
 #include <boost/iostreams/filter/zlib.hpp>
-#include <boost/iostreams/pipeline.hpp>         
+#include <boost/iostreams/pipeline.hpp>     
+#include <boost/iostreams/putback.hpp>         
 
 // Must come last.
 #if defined(BOOST_MSVC)
@@ -47,6 +49,8 @@
 #endif
 
 namespace boost { namespace iostreams {
+                    
+//------------------Definitions of constants----------------------------------//
 
 namespace gzip {
 
@@ -59,6 +63,7 @@
 const int bad_length        = 3; // Recorded length doesn't match data.
 const int bad_header        = 4; // Malformed header.
 const int bad_footer        = 5; // Malformed footer.
+const int bad_method        = 6; // Unsupported compression method.
 
 namespace magic {
 
@@ -118,6 +123,8 @@
 
 } // End namespace gzip.
 
+//------------------Definition of gzip_params---------------------------------//
+
 //
 // Class name: gzip_params.
 // Description: Subclass of zlib_params with an additional field
@@ -142,6 +149,8 @@
     std::time_t  mtime;
 };
 
+//------------------Definition of gzip_error----------------------------------//
+
 //
 // Class name: gzip_error.
 // Description: Subclass of std::ios_base::failure thrown to indicate
@@ -163,6 +172,8 @@
     int zlib_error_code_;
 };
 
+//------------------Definition of gzip_compressor-----------------------------//
+
 //
 // Template name: gzip_compressor
 // Description: Model of OutputFilter implementing compression in the
@@ -283,7 +294,7 @@
         flags_ = 0;
     }
 
-    enum flag_type {
+    enum state_type {
         f_header_done = 1,
         f_body_done = f_header_done << 1,
         f_footer_done = f_body_done << 1
@@ -297,6 +308,81 @@
 
 typedef basic_gzip_compressor<> gzip_compressor;
 
+//------------------Definition of helper templates for decompression----------//
+
+namespace detail {
+
+// Processes gzip headers
+class gzip_header {
+public:
+    gzip_header() { reset(); }
+
+    // Members for processing header data
+    void process(char c);
+    bool done() const { return state_ == s_done; }
+    void reset();
+
+    // Members for accessing header data
+    std::string file_name() const { return file_name_; }
+    std::string comment() const { return comment_; }
+    bool text() const { return (flags_ & gzip::flags::text) != 0; }
+    int os() const { return os_; }
+    std::time_t mtime() const { return mtime_; }
+private:
+    enum state_type {
+        s_id1       = 1,
+        s_id2       = s_id1 + 1,
+        s_cm        = s_id2 + 1,
+        s_flg       = s_cm + 1,
+        s_mtime     = s_flg + 1,
+        s_xfl       = s_mtime + 1,
+        s_os        = s_xfl + 1,
+        s_xlen      = s_os + 1,
+        s_extra     = s_xlen + 1,
+        s_name      = s_extra + 1,
+        s_comment   = s_name + 1,
+        s_hcrc      = s_comment + 1,
+        s_done      = s_hcrc + 1
+    };
+    std::string  file_name_;
+    std::string  comment_;
+    int          os_;
+    std::time_t  mtime_;
+    int          flags_;
+    int          state_;
+    int          offset_;  // Offset within fixed-length region.
+    int          xlen_;    // Bytes remaining in extra field.
+};
+
+// Processes gzip footers
+class gzip_footer {
+public:
+    gzip_footer() { reset(); }
+    
+    // Members for processing footer data
+    void process(char c);
+    bool done() const { return state_ == s_done; }
+    void reset();
+    
+    // Members for accessing footer data
+    zlib::ulong crc() const { return crc_; }
+    zlib::ulong uncompressed_size() const { return isize_; }
+private:
+    enum state_type {
+        s_crc     = 1,
+        s_isize   = s_crc + 1,
+        s_done    = s_isize + 1
+    };
+    zlib::ulong  crc_;
+    zlib::ulong  isize_;
+    int          state_;
+    int          offset_; 
+};
+
+} // End namespace boost::iostreams::detail.
+
+//------------------Definition of basic_gzip_decompressor---------------------//
+
 //
 // Template name: basic_gzip_decompressor
 // Description: Model of InputFilter implementing compression in the
@@ -304,6 +390,9 @@
 //
 template<typename Alloc = std::allocator<char> >
 class basic_gzip_decompressor : basic_zlib_decompressor<Alloc> {
+private:
+    typedef basic_zlib_decompressor<Alloc>   base_type;
+    typedef typename base_type::string_type  string_type;
 public:
     typedef char char_type;
     struct category
@@ -316,35 +405,70 @@
     template<typename Source>
     std::streamsize read(Source& src, char_type* s, std::streamsize n)
     {
-        if ((flags_ & f_header_read) == 0) {
-            non_blocking_adapter<Source> nb(src);
-            read_header(nb);
-            flags_ |= f_header_read;
-        }
-
-        if ((flags_ & f_footer_read) != 0)
-            return -1;
-        
-        try {
-            std::streamsize result = 0;
-            std::streamsize amt;
-            if ((amt = base_type::read(src, s, n)) != -1) {
-                result += amt;
-                if (amt < n) { // Double check for EOF.
-                    amt = base_type::read(src, s + result, n - result);
-                    if (amt != -1)
+        typedef char_traits<char>  traits_type;
+        std::streamsize            result = 0;
+        peekable_source<Source>    peek(src, putback_);
+        while (result < n && state_ != s_done) {
+            if (state_ == s_start) {
+                state_ = s_header;
+                header_.reset();
+                footer_.reset();
+            }
+            if (state_ == s_header) {
+                int c = boost::iostreams::get(peek);
+                if (traits_type::is_eof(c)) {
+                    throw gzip_error(gzip::bad_header);
+                } else if (traits_type::would_block(c)) {
+                    break;
+                }
+                header_.process(c);
+                if (header_.done())
+                    state_ = s_body;
+            } else if (state_ == s_body) {
+                try {
+                    std::streamsize amt = 
+                        base_type::read(peek, s + result, n - result);
+                    if (amt != -1) {
                         result += amt;
+                        if (amt < n - result)
+                            break;
+                    } else {
+                        peek.putback(this->unconsumed_input());
+                        state_ = s_footer;
+                    }
+                } catch (const zlib_error& e) {
+                    throw gzip_error(e);
+                }
+            } else { // state_ == s_footer
+                int c = boost::iostreams::get(peek);
+                if (traits_type::is_eof(c)) {
+                    throw gzip_error(gzip::bad_footer);
+                } else if (traits_type::would_block(c)) {
+                    break;
+                }
+                footer_.process(c);
+                if (footer_.done()) {
+                    int c = boost::iostreams::get(peek);
+                    if (traits_type::is_eof(c)) {
+                        state_ = s_done;
+                    } else {
+                        peek.putback(c);
+                        base_type::close(peek, BOOST_IOS::in);
+                        state_ = s_start;
+                        header_.reset();
+                        footer_.reset();
+                    }
                 }
             }
-            if (amt == -1) {
-                non_blocking_adapter<Source> nb(src);
-                read_footer(nb);
-                flags_ |= f_footer_read;
-            }
-            return result;
-        } catch (const zlib_error& e) {
-            throw gzip_error(e);
         }
+        if (peek.has_unconsumed_input()) {
+            putback_ = peek.unconsumed_input();
+        } else {
+            putback_.clear();
+        }
+        return result != 0 || state_ != s_done ?
+            result :
+            -1;
     }
 
     template<typename Source>
@@ -353,136 +477,95 @@
         try {
             base_type::close(src, BOOST_IOS::in);
         } catch (const zlib_error& e) {
-            flags_ = 0;
+            state_ = s_start;
+            header_.reset();
+            footer_.reset();
             throw gzip_error(e);
         }
-        flags_ = 0;
+        state_ = s_start;
     }
 
-    std::string file_name() const { return file_name_; }
-    std::string comment() const { return comment_; }
-    bool text() const { return (flags_ & gzip::flags::text) != 0; }
-    int os() const { return os_; }
-    std::time_t mtime() const { return mtime_; }
+    std::string file_name() const { return header_.file_name(); }
+    std::string comment() const { return header_.comment(); }
+    bool text() const { return header_.text(); }
+    int os() const { return header_.os(); }
+    std::time_t mtime() const { return header_.mtime(); }
 private:
-    typedef basic_zlib_decompressor<Alloc>     base_type;
-    typedef BOOST_IOSTREAMS_CHAR_TRAITS(char)  traits_type;
-    static bool is_eof(int c) { return traits_type::eq_int_type(c, EOF); }
     static gzip_params make_params(int window_bits);
 
+    // Source adapter allowing an arbitrary character sequence to be put back.
     template<typename Source>
-    static uint8_t read_uint8(Source& src, int error)
-     {
-        int c;
-        if ((c = boost::iostreams::get(src)) == EOF || c == WOULD_BLOCK)
-            throw gzip_error(error);
-        return static_cast<uint8_t>(traits_type::to_char_type(c));
-    }
+    struct peekable_source {
+        typedef char char_type;
+        struct category : source_tag, peekable_tag { };
+        explicit peekable_source(Source& src, const string_type& putback = "") 
+            : src_(src), putback_(putback), offset_(0)
+            { }
+        std::streamsize read(char* s, std::streamsize n)
+        {
+            std::streamsize result = 0;
 
-    template<typename Source>
-    static uint32_t read_uint32(Source& src, int error)
-    {
-        uint8_t b1 = read_uint8(src, error);
-        uint8_t b2 = read_uint8(src, error);
-        uint8_t b3 = read_uint8(src, error);
-        uint8_t b4 = read_uint8(src, error);
-        return b1 + (b2 << 8) + (b3 << 16) + (b4 << 24);
-    }
+            // Copy characters from putback buffer
+            std::streamsize pbsize = 
+                static_cast<std::streamsize>(putback_.size());
+            if (offset_ < pbsize) {
+                result = (std::min)(n, pbsize - offset_);
+                BOOST_IOSTREAMS_CHAR_TRAITS(char)::copy(
+                    s, putback_.data() + offset_, result);
+                offset_ += result;
+                if (result == n)
+                    return result;
+            }
 
-    template<typename Source>
-    std::string read_string(Source& src)
-    {
-        std::string result;
-        while (true) {
-            int c;
-            if (is_eof(c = boost::iostreams::get(src)))
-                throw gzip_error(gzip::bad_header);
-            else if (c == 0)
-                return result;
-            else
-                result += static_cast<char>(c);
+            // Read characters from src_
+            std::streamsize amt = 
+                boost::iostreams::read(src_, s + result, n - result);
+            return amt != -1 ? 
+                result + amt : 
+                result ? result : -1;
+        }
+        bool putback(char c)
+        {
+            if (offset_) {
+                putback_[--offset_] = c;
+                return true;
+            } else {
+                return boost::iostreams::putback(src_, c);
+            }
+        }
+        void putback(const string_type& s)
+        {
+            putback_.replace(0, offset_, s);
+            offset_ = 0;
         }
-    }
-
-    template<typename Source>
-    void read_header(Source& src) // Source is non-blocking.
-    {
-        // Reset saved values.
-        #if BOOST_WORKAROUND(__GNUC__, == 2) && defined(__STL_CONFIG_H) || \
-            BOOST_WORKAROUND(BOOST_DINKUMWARE_STDLIB, == 1) \
-            /**/
-            file_name_.erase(0, std::string::npos);
-            comment_.erase(0, std::string::npos);
-        #else
-            file_name_.clear();
-            comment_.clear();
-        #endif
-        os_ = gzip::os_unknown;
-        mtime_ = 0;
-
-        int flags;
 
-        // Read header, without checking header crc.
-        if ( boost::iostreams::get(src) != gzip::magic::id1 ||   // ID1.
-             boost::iostreams::get(src) != gzip::magic::id2 ||   // ID2.
-             is_eof(boost::iostreams::get(src)) ||               // CM.
-             is_eof(flags = boost::iostreams::get(src)) )        // FLG.
+        // Returns true if some characters have been putback but not re-read.
+        bool has_unconsumed_input() const 
         {
-            throw gzip_error(gzip::bad_header);
+            return offset_ < static_cast<std::streamsize>(putback_.size());
         }
-        mtime_ = read_uint32(src, gzip::bad_header);        // MTIME.
-        read_uint8(src, gzip::bad_header);                 // XFL.
-        os_ = read_uint8(src, gzip::bad_header);          // OS.
-        if (flags & boost::iostreams::gzip::flags::text)
-            flags_ |= f_text;
-
-        // Skip extra field. (From J. Halleaux; see note at top.)
-        if (flags & gzip::flags::extra) {
-            int length = 
-                static_cast<int>(
-                    read_uint8(src, gzip::bad_header) +
-                    (read_uint8(src, gzip::bad_header) << 8)
-                );
-            // length is garbage if EOF but the loop below will quit anyway.
-            do { }
-            while (length-- != 0 && !is_eof(boost::iostreams::get(src)));
-        }
-
-        if (flags & gzip::flags::name)          // Read file name.
-            file_name_ = read_string(src);
-        if (flags & gzip::flags::comment)       // Read comment.
-            comment_ = read_string(src);
-        if (flags & gzip::flags::header_crc) {  // Skip header crc.
-            read_uint8(src, gzip::bad_header);
-            read_uint8(src, gzip::bad_header);
+
+        // Returns the sequence of characters that have been put back but not re-read.
+        string_type unconsumed_input() const
+        {
+            return string_type(putback_, offset_, putback_.size() - offset_);
         }
-    }
+        Source&          src_;
+        string_type      putback_;
+        std::streamsize  offset_;
+    };
 
-    template<typename Source>
-    void read_footer(Source& src)
-    {
-        typename base_type::string_type footer = 
-            this->unconsumed_input();
-        int c;
-        while (!is_eof(c = boost::iostreams::get(src)))
-            footer += c;
-        detail::range_adapter<input, std::string> 
-            rng(footer.begin(), footer.end());
-        if (read_uint32(rng, gzip::bad_footer) != this->crc())
-            throw gzip_error(gzip::bad_crc);
-        if (static_cast<int>(read_uint32(rng, gzip::bad_footer)) != this->total_out())
-            throw gzip_error(gzip::bad_length);
-    }
-    enum flag_type {
-        f_header_read  = 1,
-        f_footer_read  = f_header_read << 1,
-        f_text         = f_footer_read << 1
+    enum state_type {
+        s_start   = 1,
+        s_header  = s_start + 1,
+        s_body    = s_header + 1,
+        s_footer  = s_body + 1,
+        s_done    = s_footer + 1
     };
-    std::string  file_name_;
-    std::string  comment_;
-    int          os_;
-    std::time_t  mtime_;
-    int          flags_;
+    detail::gzip_header  header_;
+    detail::gzip_footer  footer_;
+    string_type          putback_;
+    int                  state_;
 };
 BOOST_IOSTREAMS_PIPABLE(basic_gzip_decompressor, 1)
 
@@ -574,13 +657,164 @@
     return amt;
 }
 
+//------------------Implementation of gzip_header-----------------------------//
+
+namespace detail {
+
+void gzip_header::process(char c)
+{
+    uint8_t value = static_cast<uint8_t>(c);
+    switch (state_) {
+    case s_id1:
+        if (value != gzip::magic::id1)
+            throw gzip_error(gzip::bad_header);
+        state_ = s_id2;
+        break;
+    case s_id2:
+        if (value != gzip::magic::id2)
+            throw gzip_error(gzip::bad_header);
+        state_ = s_cm;
+        break;
+    case s_cm:
+        if (value != gzip::method::deflate)
+            throw gzip_error(gzip::bad_method);
+        state_ = s_flg;
+        break;
+    case s_flg:
+        flags_ = value;
+        state_ = s_mtime;
+        break;
+    case s_mtime:
+        mtime_ += value << (offset_ * 8);
+        if (offset_ == 3) {
+            state_ = s_xfl;
+            offset_ = 0;
+        } else {
+            ++offset_;
+        }
+        break;
+    case s_xfl:
+        state_ = s_os;
+        break;
+    case s_os:
+        os_ = value;
+        if (flags_ & gzip::flags::extra) {
+            state_ = s_extra;
+        } else if (flags_ & gzip::flags::name) {
+            state_ = s_name;
+        } else if (flags_ & gzip::flags::comment) {
+            state_ = s_comment;
+        } else if (flags_ & gzip::flags::header_crc) {
+            state_ = s_hcrc;
+        } else {
+            state_ = s_done;
+        }
+        break;
+    case s_xlen:
+        xlen_ += value << (offset_ * 8);
+        if (offset_ == 1) {
+            state_ = s_extra;
+            offset_ = 0;
+        } else {
+            ++offset_;
+        }
+        break;
+    case s_extra:
+        if (--xlen_ == 0) {
+            if (flags_ & gzip::flags::name) {
+                state_ = s_name;
+            } else if (flags_ & gzip::flags::comment) {
+                state_ = s_comment;
+            } else if (flags_ & gzip::flags::header_crc) {
+                state_ = s_hcrc;
+            } else {
+                state_ = s_done;
+            }
+        }
+        break;
+    case s_name:
+        if (c != 0) {
+            file_name_ += c;
+        } else if (flags_ & gzip::flags::comment) {
+            state_ = s_comment;
+        } else if (flags_ & gzip::flags::header_crc) {
+            state_ = s_hcrc;
+        } else {
+            state_ = s_done;
+        }
+        break;
+    case s_comment:
+        if (c != 0) {
+            comment_ += c;
+        } else if (flags_ & gzip::flags::header_crc) {
+            state_ = s_hcrc;
+        } else {
+            state_ = s_done;
+        }
+        break;
+    case s_hcrc:
+        if (offset_ == 1) {
+            state_ = s_done;
+            offset_ = 0;
+        } else {
+            ++offset_;
+        }
+        break;
+    default:
+        assert(0);
+    }
+}
+
+void gzip_header::reset()
+{
+    file_name_.clear();
+    comment_.clear();
+    os_ = flags_ = offset_ = xlen_ = 0;
+    mtime_ = 0;
+    state_ = s_id1;
+}
+
+//------------------Implementation of gzip_footer-----------------------------//
+
+void gzip_footer::process(char c)
+{
+    uint8_t value = static_cast<uint8_t>(c);
+    if (state_ == s_crc) {
+        crc_ += value << (offset_ * 8);
+        if (offset_ == 3) {
+            state_ = s_isize;
+            offset_ = 0;
+        } else {
+            ++offset_;
+        }
+    } else if (state_ = s_isize) {
+        isize_ += value << (offset_ * 8);
+        if (offset_ == 3) {
+            state_ = s_done;
+            offset_ = 0;
+        } else {
+            ++offset_;
+        }
+    } else {
+        assert(0);
+    }
+}
+
+void gzip_footer::reset()
+{
+    state_ = s_crc;
+    offset_ = 0;
+}
+
+} // End namespace boost::iostreams::detail.
+
 //------------------Implementation of gzip_decompressor-----------------------//
 
 template<typename Alloc>
 basic_gzip_decompressor<Alloc>::basic_gzip_decompressor
     (int window_bits, int buffer_size)
     : base_type(make_params(window_bits), buffer_size),
-      os_(gzip::os_unknown), mtime_(0), flags_(0)
+      state_(s_start)
     { }
 
 template<typename Alloc>

Modified: trunk/libs/iostreams/test/gzip_test.cpp
==============================================================================
--- trunk/libs/iostreams/test/gzip_test.cpp	(original)
+++ trunk/libs/iostreams/test/gzip_test.cpp	2008-05-31 18:53:58 EDT (Sat, 31 May 2008)
@@ -6,8 +6,14 @@
 // See http://www.boost.org/libs/iostreams for documentation.
 
 #include <string>
+#include <boost/iostreams/copy.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <boost/iostreams/device/back_inserter.hpp>
 #include <boost/iostreams/filter/gzip.hpp>
 #include <boost/iostreams/filter/test.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+#include <boost/ref.hpp>
+#include <boost/range/iterator_range.hpp>
 #include <boost/test/test_tools.hpp>
 #include <boost/test/unit_test.hpp>
 #include "detail/sequence.hpp"
@@ -16,18 +22,36 @@
 using namespace boost;
 using namespace boost::iostreams;
 using namespace boost::iostreams::test;
+namespace io = boost::iostreams;
 using boost::unit_test::test_suite;     
 
 struct gzip_alloc : std::allocator<char> { };
 
-void gzip_test()
+void compression_test()
 {
-    text_sequence data;
-    BOOST_CHECK(
-        test_filter_pair( gzip_compressor(), 
-                          gzip_decompressor(), 
-                          std::string(data.begin(), data.end()) )
-    );
+    text_sequence      data;
+
+    // Test compression and decompression with metadata
+    for (int i = 0; i < 4; ++i) {
+        gzip_params params;
+        if (i & 1) {
+            params.file_name = "original file name";
+        }
+        if (i & 2) {
+            params.comment = "detailed file description";
+        }
+        gzip_compressor    out(params);
+        gzip_decompressor  in;
+        BOOST_CHECK(
+            test_filter_pair( boost::ref(out), 
+                              boost::ref(in), 
+                              std::string(data.begin(), data.end()) )
+        );
+        BOOST_CHECK(in.file_name() == params.file_name);
+        BOOST_CHECK(in.comment() == params.comment);
+    }
+
+    // Test compression and decompression with custom allocator
     BOOST_CHECK(
         test_filter_pair( basic_gzip_compressor<gzip_alloc>(), 
                           basic_gzip_decompressor<gzip_alloc>(), 
@@ -35,9 +59,34 @@
     );
 }
 
+void multiple_member_test()
+{
+    text_sequence      data;
+    std::vector<char>  temp, dest;
+
+    // Write compressed data to temp, twice in succession
+    filtering_ostream out;
+    out.push(gzip_compressor());
+    out.push(io::back_inserter(temp));
+    io::copy(make_iterator_range(data), out);
+    out.push(io::back_inserter(temp));
+    io::copy(make_iterator_range(data), out);
+
+    // Read compressed data from temp into dest
+    filtering_istream in;
+    in.push(gzip_decompressor());
+    in.push(array_source(&temp[0], temp.size()));
+    io::copy(in, io::back_inserter(dest));
+
+    // Check that dest consists of two copies of data
+    BOOST_CHECK(std::equal(data.begin(), data.end(), dest.begin()));
+    BOOST_CHECK(std::equal(data.begin(), data.end(), dest.begin() + dest.size() / 2));
+}
+
 test_suite* init_unit_test_suite(int, char* []) 
 {
     test_suite* test = BOOST_TEST_SUITE("gzip test");
-    test->add(BOOST_TEST_CASE(&gzip_test));
+    test->add(BOOST_TEST_CASE(&compression_test));
+    test->add(BOOST_TEST_CASE(&multiple_member_test));
     return test;
 }