[Boost-commit] svn:boost r84150 - in trunk/tools/quickbook: src test/unit

Subject: [Boost-commit] svn:boost r84150 - in trunk/tools/quickbook: src test/unit
From: dnljms_at_[hidden]
Date: 2013-05-05 09:46:24

Author: danieljames
Date: 2013-05-05 09:46:23 EDT (Sun, 05 May 2013)
New Revision: 84150
URL: http://svn.boost.org/trac/boost/changeset/84150

Log:
Clean up mixed tabs/spaces when unindenting.

If the whitespace to be removed when unindenting a block is not
consistent, then replace all the indentation in that block with spaces
(i.e. expand tabs). This fixes some odd results that could happen, e.g.

    ....Line 1
    --->Line 2

(dots = spaces, arrow = tab) would become:

    ...Line 1
    Line 2

Because the first character was removed from each line. This now works
okay.

But I don't expand tabs when the whitespace that isn't removed is
inconsistent, which can have odd results, for example:

    ..Line 1
    ..->Line 2
    ......Line 3

Will just remove the first two spaces, resulting in:

    Line 1
    --->Line 2
    ....Line 3

This should be rare, as tabs after spaces are a bit odd. But maybe I
should convert to spaces more often. Perhaps when indentation is
inconsistent, or perhaps whenever a mixture of tabs and spaces are used,
or even always just do it.
Text files modified: 
   trunk/tools/quickbook/src/files.cpp                 |   147 ++++++++++++++++++++++++++++++--------- 
   trunk/tools/quickbook/test/unit/source_map_test.cpp |    40 ++++++++++                              
   2 files changed, 151 insertions(+), 36 deletions(-)

Modified: trunk/tools/quickbook/src/files.cpp
==============================================================================
--- trunk/tools/quickbook/src/files.cpp	(original)
+++ trunk/tools/quickbook/src/files.cpp	2013-05-05 09:46:23 EDT (Sun, 05 May 2013)
@@ -468,68 +468,143 @@
         }
     }
 
+    boost::string_ref::size_type indentation_count(boost::string_ref x)
+    {
+        unsigned count = 0;
+
+        for(boost::string_ref::const_iterator begin = x.begin(), end = x.end();
+            begin != end; ++begin)
+        {
+            switch(*begin)
+            {
+            case ' ':
+                ++count;
+                break;
+            case '\t':
+                // hardcoded tab to 4 for now
+                count = count - (count % 4) + 4;
+                break;
+            default:
+                assert(false);
+            }
+        }
+
+        return count;
+    }
+
     void mapped_file_builder::unindent_and_add(boost::string_ref x)
     {
-        std::string program(x.begin(), x.end());
+        // I wanted to do everything using a string_ref, but unfortunately
+        // they don't have all the overloads used in here. So...
+        std::string const program(x.begin(), x.end());
 
         // Erase leading blank lines and newlines:
         std::string::size_type start = program.find_first_not_of(" \t\r\n");
         if (start == std::string::npos) return;
 
         start = program.find_last_of("\r\n", start);
-        if (start != std::string::npos)
-        {
-            ++start;
-            program.erase(0, start);
-        }
+        start = start == std::string::npos ? 0 : start + 1;
 
-        assert(program.size() != 0);
+        assert(start < program.size());
 
-        // Get the first line indent
-        std::string::size_type indent = program.find_first_not_of(" \t");
-        std::string::size_type pos = 0;
-        if (std::string::npos == indent)
-        {
-            // Nothing left to do here. The code is empty (just spaces).
-            // We clear the program to signal the caller that it is empty
-            // and return early.
-            program.clear();
-            return;
-        }
+        // Get the first line indentation
+        std::string::size_type indent = program.find_first_not_of(" \t", start) - start;
+        boost::string_ref::size_type full_indent = indentation_count(
+            boost::string_ref(&program[start], indent));
+
+        std::string::size_type pos = start;
 
         // Calculate the minimum indent from the rest of the lines
-        do
+        // Detecting a mix of spaces and tabs.
+        while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
         {
             pos = program.find_first_not_of("\r\n", pos);
-            if (std::string::npos == pos)
-                break;
+            if (std::string::npos == pos) break;
 
             std::string::size_type n = program.find_first_not_of(" \t", pos);
-            if (n != std::string::npos)
-            {
-                char ch = program[n];
-                if (ch != '\r' && ch != '\n') // ignore empty lines
-                    indent = (std::min)(indent, n-pos);
-            }
+            if (n == std::string::npos) break;
+
+            char ch = program[n];
+            if (ch == '\r' || ch == '\n') continue; // ignore empty lines
+
+            indent = (std::min)(indent, n-pos);
+            full_indent = (std::min)(full_indent, indentation_count(
+                boost::string_ref(&program[pos], n-pos)));
         }
-        while (std::string::npos != (pos = program.find_first_of("\r\n", pos)));
 
-        // Trim white spaces from column 0..indent
-        pos = 0;
-        program.erase(0, indent);
+        // Detect if indentation is mixed.
+        bool mixed_indentation = false;
+        boost::string_ref first_indent(&program[start], indent);
+        pos = start;
+
         while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
         {
-            if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
-            {
+            pos = program.find_first_not_of("\r\n", pos);
+            if (std::string::npos == pos) break;
+
+            std::string::size_type n = program.find_first_not_of(" \t", pos);
+            if (n == std::string::npos || n-pos < indent) continue;
+
+            if (boost::string_ref(&program[pos], indent) != first_indent) {
+                mixed_indentation = true;
                 break;
             }
+        }
+
+        std::string unindented_program;
+        std::string::size_type copied = start;
+
+        if (mixed_indentation)
+        {
+            pos = start;
+
+            do {
+                if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
+                    break;
+
+                unindented_program.append(program.begin() + copied, program.begin() + pos);
+                copied = pos;
 
-            std::string::size_type next = program.find_first_of("\r\n", pos);
-            program.erase(pos, (std::min)(indent, next-pos));
+                std::string::size_type next = program.find_first_not_of(" \t", pos);
+
+                unsigned length = indentation_count(boost::string_ref(
+                    &program[pos], next - pos));
+
+                if (length > full_indent) {
+                    std::string new_indentation(length - full_indent, ' ');
+                    unindented_program.append(new_indentation);
+                }
+
+                copied = next;
+            } while (std::string::npos !=
+                (pos = program.find_first_of("\r\n", pos)));
         }
+        else
+        {
+            // Trim white spaces from column 0..indent
+            pos = start + indent;
+            copied = pos;
+
+            while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
+            {
+                if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
+                {
+                    break;
+                }
+
+                unindented_program.append(program.begin() + copied, program.begin() + pos);
+                copied = pos;
+
+                std::string::size_type next = program.find_first_of("\r\n", pos);
+                copied = pos + (std::min)(indent, next-pos);
+            }
+        }
+
+        unindented_program.append(program.begin() + copied, program.end());
+        copied = program.size();
 
         data->new_file->add_indented_mapped_file_section(x.begin());
-        data->new_file->source_.append(program);
+        data->new_file->source_.append(unindented_program);
     }
 
     file_position mapped_file::position_of(boost::string_ref::const_iterator pos) const

Modified: trunk/tools/quickbook/test/unit/source_map_test.cpp
==============================================================================
--- trunk/tools/quickbook/test/unit/source_map_test.cpp	(original)
+++ trunk/tools/quickbook/test/unit/source_map_test.cpp	2013-05-05 09:46:23 EDT (Sun, 05 May 2013)
@@ -306,11 +306,51 @@
 
 }
 
+void indented_map_mixed_test()
+{
+    quickbook::mapped_file_builder builder;
+
+    {
+        boost::string_ref source("\tCode line 1\n    Code line 2\n\t    Code line 3\n    \tCode line 4");
+        quickbook::file_ptr fake_file = new quickbook::file(
+            "(fake file)", source, 105u);
+        builder.start(fake_file);
+        builder.unindent_and_add(fake_file->source());
+        quickbook::file_ptr f1 = builder.release();
+        BOOST_TEST_EQ(f1->source(),
+            boost::string_ref("Code line 1\nCode line 2\n    Code line 3\n    Code line 4"));
+    }
+
+    {
+        boost::string_ref source("  Code line 1\n\tCode line 2");
+        quickbook::file_ptr fake_file = new quickbook::file(
+            "(fake file)", source, 105u);
+        builder.start(fake_file);
+        builder.unindent_and_add(fake_file->source());
+        quickbook::file_ptr f1 = builder.release();
+        BOOST_TEST_EQ(f1->source(),
+            boost::string_ref("Code line 1\n  Code line 2"));
+    }
+
+    {
+        boost::string_ref source("  Code line 1\n  \tCode line 2");
+        quickbook::file_ptr fake_file = new quickbook::file(
+            "(fake file)", source, 105u);
+        builder.start(fake_file);
+        builder.unindent_and_add(fake_file->source());
+        quickbook::file_ptr f1 = builder.release();
+        BOOST_TEST_EQ(f1->source(),
+            boost::string_ref("Code line 1\n\tCode line 2"));
+    }
+}
+
+
 int main()
 {
     simple_map_tests();
     indented_map_tests();
     indented_map_tests2();
     indented_map_leading_blanks_test();
+    indented_map_mixed_test();
     return boost::report_errors();
 }