[Boost-commit] svn:boost r86699 - in trunk/tools/quickbook: doc src test/unit

Subject: [Boost-commit] svn:boost r86699 - in trunk/tools/quickbook: doc src test/unit
From: dnljms_at_[hidden]
Date: 2013-11-14 14:19:54

Author: danieljames
Date: 2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)
New Revision: 86699
URL: http://svn.boost.org/trac/boost/changeset/86699

Log:
Add glob support.

This is based on Rene's implementation, but I used my own glob function,
and adjusted a few things since it's now always using ascii. It would be
nice to support unicode, but that would require at the very least a
normalization library, and perhaps more than that.

Added:
   trunk/tools/quickbook/src/glob.cpp   (contents, props changed)
   trunk/tools/quickbook/src/glob.hpp   (contents, props changed)
   trunk/tools/quickbook/test/unit/glob_test.cpp   (contents, props changed)
Text files modified: 
   trunk/tools/quickbook/doc/1_7.qbk             |    20 +++++                                   
   trunk/tools/quickbook/src/Jamfile.v2          |     1                                         
   trunk/tools/quickbook/src/actions.cpp         |    10 ++                                      
   trunk/tools/quickbook/src/glob.cpp            |   154 +++++++++++++++++++++++++++++++++++++++ 
   trunk/tools/quickbook/src/glob.hpp            |    15 +++                                     
   trunk/tools/quickbook/src/include_paths.cpp   |   156 +++++++++++++++++++++++++++++++-------- 
   trunk/tools/quickbook/src/include_paths.hpp   |     4                                         
   trunk/tools/quickbook/test/unit/Jamfile.v2    |     1                                         
   trunk/tools/quickbook/test/unit/glob_test.cpp |    97 ++++++++++++++++++++++++                
   9 files changed, 423 insertions(+), 35 deletions(-)

Modified: trunk/tools/quickbook/doc/1_7.qbk
==============================================================================
--- trunk/tools/quickbook/doc/1_7.qbk	Thu Nov 14 13:53:51 2013	(r86698)
+++ trunk/tools/quickbook/doc/1_7.qbk	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -168,4 +168,24 @@
 
 [endsect]
 
+[section:glob Including multiple files with Globs]
+
+One can now include multiple files at once using a glob pattern for the
+file reference:
+
+    [include sub/*/*.qbk]
+    [include include/*.h]
+
+All the matching files, and intermediate irectories, will match and be
+included. The glob pattern can be "\*" for matching zero or more characters,
+"?" for matching a single character, "\[<c>-<c>\]" to match a character class,
+"\[\^<char>-<char>\]" to exclusive match a character class, "\\\\" to escape
+a glob special character which is then matched, and anything else is matched
+to the character.
+
+[note Because of the escaping in file references the "\\\\" glob escape is
+a double "\\"; i.e. and escaped back-slash.]
+
+[endsect]
+
 [endsect] [/ Quickbok 1.7]

Modified: trunk/tools/quickbook/src/Jamfile.v2
==============================================================================
--- trunk/tools/quickbook/src/Jamfile.v2	Thu Nov 14 13:53:51 2013	(r86698)
+++ trunk/tools/quickbook/src/Jamfile.v2	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -31,6 +31,7 @@
     utils.cpp
     files.cpp
     native_text.cpp
+    glob.cpp
     include_paths.cpp
     values.cpp
     document_state.cpp

Modified: trunk/tools/quickbook/src/actions.cpp
==============================================================================
--- trunk/tools/quickbook/src/actions.cpp	Thu Nov 14 13:53:51 2013	(r86698)
+++ trunk/tools/quickbook/src/actions.cpp	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -1863,6 +1863,16 @@
     {
         path_parameter parameter = check_path(p, state);
 
+        if (parameter.type == path_parameter::glob) {
+            // TODO: Should know if this is an xinclude or an xmlbase.
+            // Would also help with implementation of 'check_path'.
+            detail::outerr(p.get_file(), p.get_position())
+                << "Glob used in xinclude/xmlbase."
+                << std::endl;
+            ++state.error_count;
+            return xinclude_path(state.current_file->path.parent_path(), "");
+        }
+
         fs::path path = detail::generic_to_path(parameter.value);
         fs::path full_path = path;
 

Added: trunk/tools/quickbook/src/glob.cpp
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ trunk/tools/quickbook/src/glob.cpp	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -0,0 +1,154 @@
+/*=============================================================================
+    Copyright (c) 2013 Daniel James
+
+    Use, modification and distribution is subject to the Boost Software
+    License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+    http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+
+#include "glob.hpp"
+#include <cassert>
+
+namespace quickbook
+{
+    typedef boost::string_ref::const_iterator glob_iterator;
+
+    bool match_section(glob_iterator& pattern_begin, glob_iterator pattern_end,
+            glob_iterator& filename_begin, glob_iterator& filename_end);
+    bool match_range(glob_iterator& pattern_begin, glob_iterator pattern_end,
+            unsigned char x);
+
+    bool glob(boost::string_ref const& pattern,
+            boost::string_ref const& filename)
+    {
+        // If there wasn't this special case then '*' would match an
+        // empty string.
+        if (filename.empty()) return pattern.empty();
+
+        glob_iterator pattern_it = pattern.begin();
+        glob_iterator pattern_end = pattern.end();
+
+        glob_iterator filename_it = filename.begin();
+        glob_iterator filename_end = filename.end();
+
+        if (!match_section(pattern_it, pattern_end, filename_it, filename_end))
+            return false;
+
+        while (pattern_it != pattern_end) {
+            assert(*pattern_it == '*');
+            ++pattern_it;
+            if (pattern_it == pattern_end) return true;
+
+            // TODO: Error?
+            if (*pattern_it == '*') return false;
+
+            while (true) {
+                if (filename_it == filename_end) return false;
+                if (match_section(pattern_it, pattern_end, filename_it, filename_end))
+                    break;
+                ++filename_it;
+            }
+        }
+
+        return filename_it == filename_end;
+    }
+
+    bool match_section(glob_iterator& pattern_begin, glob_iterator pattern_end,
+            glob_iterator& filename_begin, glob_iterator& filename_end)
+    {
+        glob_iterator pattern_it = pattern_begin;
+        glob_iterator filename_it = filename_begin;
+
+        while (pattern_it != pattern_end && *pattern_it != '*') {
+            if (filename_it == filename_end) return false;
+
+            switch(*pattern_it) {
+                case '*':
+                    assert(false);
+                    return false;
+                case '[':
+                    if (!match_range(pattern_it, pattern_end, *filename_it))
+                        return false;
+                    ++filename_it;
+                    break;
+                case '?':
+                    ++pattern_it;
+                    ++filename_it;
+                    break;
+                case '\\':
+                    ++pattern_it;
+                    if (pattern_it == pattern_end) return false;
+                    BOOST_FALLTHROUGH;
+                default:
+                    if (*pattern_it != *filename_it) return false;
+                    ++pattern_it;
+                    ++filename_it;
+            }
+        }
+
+        if (pattern_it == pattern_end && filename_it != filename_end)
+            return false;
+
+        pattern_begin = pattern_it;
+        filename_begin = filename_it;
+        return true;
+    }
+
+    bool match_range(glob_iterator& pattern_begin, glob_iterator pattern_end,
+            unsigned char x)
+    {
+        assert(pattern_begin != pattern_end && *pattern_begin == '[');
+        ++pattern_begin;
+        if (pattern_begin == pattern_end) return false;
+
+        bool invert_match = false;
+        bool matched = false;
+
+        if (*pattern_begin == '^') {
+            invert_match = true;
+            ++pattern_begin;
+            if (pattern_begin == pattern_end) return false;
+        }
+
+        // Search for a match
+        while (true) {
+            unsigned char first = *pattern_begin;
+            ++pattern_begin;
+            if (first == ']') break;
+            if (pattern_begin == pattern_end) return false;
+
+            if (first == '\\') {
+                first = *pattern_begin;
+                ++pattern_begin;
+                if (pattern_begin == pattern_end) return false;
+            }
+
+            if (*pattern_begin != '-') {
+                matched = matched || (first == x);
+            }
+            else {
+                ++pattern_begin;
+                if (pattern_begin == pattern_end) return false;
+
+                unsigned char second = *pattern_begin;
+                ++pattern_begin;
+                if (second == ']') {
+                    matched = matched || (first == x) || (x == '-');
+                    break;
+                }
+                if (pattern_begin == pattern_end) return false;
+
+                if (second == '\\') {
+                    second = *pattern_begin;
+                    ++pattern_begin;
+                    if (pattern_begin == pattern_end) return false;
+                }
+
+                // TODO: What if second < first?
+                matched = matched || (first <= x && x <= second);
+            }
+        }
+
+        return invert_match != matched;
+    }
+}

Added: trunk/tools/quickbook/src/glob.hpp
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ trunk/tools/quickbook/src/glob.hpp	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -0,0 +1,15 @@
+/*=============================================================================
+    Copyright (c) 2013 Daniel James
+
+    Use, modification and distribution is subject to the Boost Software
+    License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+    http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+
+#include <boost/utility/string_ref.hpp>
+
+namespace quickbook
+{
+    bool glob(boost::string_ref const& pattern,
+            boost::string_ref const& filename);
+}

Modified: trunk/tools/quickbook/src/include_paths.cpp
==============================================================================
--- trunk/tools/quickbook/src/include_paths.cpp	Thu Nov 14 13:53:51 2013	(r86698)
+++ trunk/tools/quickbook/src/include_paths.cpp	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -10,12 +10,15 @@
 =============================================================================*/
 
 #include "native_text.hpp"
+#include "glob.hpp"
 #include "include_paths.hpp"
 #include "state.hpp"
 #include "utils.hpp"
 #include "quickbook.hpp" // For the include_path global (yuck)
 #include <boost/foreach.hpp>
 #include <boost/range/algorithm/replace.hpp>
+#include <boost/filesystem/operations.hpp>
+#include <cassert>
 
 namespace quickbook
 {
@@ -34,7 +37,10 @@
         std::string path_text = qbk_version_n >= 106u || path.is_encoded() ?
                 path.get_encoded() : detail::to_s(path.get_quickbook());
 
-        if(path_text.find('\\') != std::string::npos)
+        bool is_glob = qbk_version_n >= 107u &&
+            path_text.find_first_of("[]?*") != std::string::npos;
+
+        if(!is_glob && path_text.find('\\') != std::string::npos)
         {
             quickbook::detail::ostream* err;
 
@@ -54,62 +60,146 @@
             boost::replace(path_text, '\\', '/');
         }
 
-        return path_parameter(path_text, path_parameter::path);
+        return path_parameter(path_text,
+            is_glob ? path_parameter::glob : path_parameter::path);
     }
 
     //
     // Search include path
     //
 
-    std::set<quickbook_path> include_search(
-            path_parameter const& parameter,
-            quickbook::state& state, string_iterator pos)
+    void include_search_glob(std::set<quickbook_path> & result,
+        fs::path dir, std::string path, quickbook::state& state)
     {
-        std::set<quickbook_path> result;
+        // Search for the first part of the path that contains glob
+        // characters. (TODO: Account for escapes?)
 
-        fs::path path = detail::generic_to_path(parameter.value);
+        std::size_t glob_pos = path.find_first_of("[]?*");
 
-        // If the path is relative, try and resolve it.
-        if (!path.has_root_directory() && !path.has_root_name())
+        if (glob_pos == std::string::npos)
         {
-            fs::path local_path =
-                state.current_file->path.parent_path() / path;
-
-            // See if it can be found locally first.
-            if (state.dependencies.add_dependency(local_path))
+            if (state.dependencies.add_dependency(dir / path))
             {
                 result.insert(quickbook_path(
-                    local_path,
+                    dir / path,
                     state.abstract_file_path.parent_path() / path));
-                return result;
             }
+            return;
+        }
+
+        std::size_t prev = path.rfind('/', glob_pos);
+        std::size_t next = path.find('/', glob_pos);
 
-            BOOST_FOREACH(fs::path full, include_path)
+        std::size_t glob_begin = prev == std::string::npos ? 0 : prev + 1;
+        std::size_t glob_end = next == std::string::npos ? path.size() : next;
+
+        if (prev != std::string::npos)
+            dir /= fs::path(path.substr(0, prev));
+
+        if (next == std::string::npos) next = path.size();
+        else ++next;
+
+        boost::string_ref glob(
+                path.data() + glob_begin,
+                glob_end - glob_begin);
+
+        // Walk through the dir for matches.
+        fs::directory_iterator dir_i(dir.empty() ? fs::path(".") : dir);
+        fs::directory_iterator dir_e;
+        for (; dir_i != dir_e; ++dir_i)
+        {
+            fs::path f = dir_i->path().filename();
+
+            // Skip if the dir item doesn't match.
+            if (!quickbook::glob(glob, detail::path_to_generic(f))) continue;
+
+            // If it's a file we add it to the results.
+            if (fs::is_regular_file(dir_i->status()))
             {
-                full /= path;
+                result.insert(quickbook_path(
+                    dir/f,
+                    state.abstract_file_path.parent_path()/dir/f
+                    ));
+            }
+            // If it's a matching dir, we recurse looking for more files.
+            else
+            {
+                include_search_glob(result, dir/f,
+                        path.substr(next), state);
+            }
+        }
+    }
 
-                if (state.dependencies.add_dependency(full))
-                {
-                    result.insert(quickbook_path(full, path));
-                    return result;
-                }
+    std::set<quickbook_path> include_search(path_parameter const& parameter,
+            quickbook::state& state, string_iterator pos)
+    {
+        std::set<quickbook_path> result;
+
+        // If the path has some glob match characters
+        // we do a discovery of all the matches..
+        if (parameter.type == path_parameter::glob)
+        {
+            fs::path current = state.current_file->path.parent_path();
+
+            // Search for the current dir accumulating to the result.
+            include_search_glob(result, current, parameter.value, state);
+
+            // Search the include path dirs accumulating to the result.
+            BOOST_FOREACH(fs::path dir, include_path)
+            {
+                include_search_glob(result, dir, parameter.value, state);
             }
+
+            // Done.
+            return result;
         }
         else
         {
-            if (state.dependencies.add_dependency(path)) {
-                result.insert(quickbook_path(path, path));
-                return result;
+            fs::path path = detail::generic_to_path(parameter.value);
+
+            // If the path is relative, try and resolve it.
+            if (!path.has_root_directory() && !path.has_root_name())
+            {
+                fs::path local_path =
+                   state.current_file->path.parent_path() / path;
+
+                // See if it can be found locally first.
+                if (state.dependencies.add_dependency(local_path))
+                {
+                    result.insert(quickbook_path(
+                        local_path,
+                        state.abstract_file_path.parent_path() / path));
+                    return result;
+                }
+
+                // Search in each of the include path locations.
+                BOOST_FOREACH(fs::path full, include_path)
+                {
+                    full /= path;
+
+                    if (state.dependencies.add_dependency(full))
+                    {
+                        result.insert(quickbook_path(full, path));
+                        return result;
+                    }
+                }
+            }
+            else
+            {
+                if (state.dependencies.add_dependency(path)) {
+                    result.insert(quickbook_path(path, path));
+                    return result;
+                }
             }
-        }
 
-        detail::outerr(state.current_file, pos)
-            << "Unable to find file: "
-            << parameter.value
-            << std::endl;
-        ++state.error_count;
+            detail::outerr(state.current_file, pos)
+                << "Unable to find file: "
+                << parameter.value
+                << std::endl;
+            ++state.error_count;
 
-        return result;
+            return result;
+        }
     }
 
     //

Modified: trunk/tools/quickbook/src/include_paths.hpp
==============================================================================
--- trunk/tools/quickbook/src/include_paths.hpp	Thu Nov 14 13:53:51 2013	(r86698)
+++ trunk/tools/quickbook/src/include_paths.hpp	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -24,8 +24,8 @@
 namespace quickbook
 {
     struct path_parameter {
-        // Will possibly add 'url' and 'glob' to this list later:
-        enum path_type { path };
+        // Will possibly add 'url' to this list later:
+        enum path_type { path, glob };
 
         std::string value;
         path_type type;

Modified: trunk/tools/quickbook/test/unit/Jamfile.v2
==============================================================================
--- trunk/tools/quickbook/test/unit/Jamfile.v2	Thu Nov 14 13:53:51 2013	(r86698)
+++ trunk/tools/quickbook/test/unit/Jamfile.v2	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -23,6 +23,7 @@
 run values_test.cpp ../../src/values.cpp ../../src/files.cpp ;
 run post_process_test.cpp ../../src/post_process.cpp ;
 run source_map_test.cpp ../../src/files.cpp ;
+run glob_test.cpp ../../src/glob.cpp ;
 
 # Copied from spirit
 run symbols_tests.cpp ;

Added: trunk/tools/quickbook/test/unit/glob_test.cpp
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ trunk/tools/quickbook/test/unit/glob_test.cpp	2013-11-14 14:19:54 EST (Thu, 14 Nov 2013)	(r86699)
@@ -0,0 +1,97 @@
+/*=============================================================================
+    Copyright (c) 2013 Daniel James
+
+    Use, modification and distribution is subject to the Boost Software
+    License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+    http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+
+#include "glob.hpp"
+#include <boost/detail/lightweight_test.hpp>
+
+void glob_tests() {
+    BOOST_TEST(quickbook::glob("", ""));
+
+    BOOST_TEST(!quickbook::glob("*", ""));
+    BOOST_TEST(quickbook::glob("*", "a"));
+    BOOST_TEST(quickbook::glob("*b", "b"));
+    BOOST_TEST(quickbook::glob("*b", "ab"));
+    BOOST_TEST(quickbook::glob("*b", "bab"));
+    BOOST_TEST(quickbook::glob("*b*", "b"));
+    BOOST_TEST(quickbook::glob("*b*", "ab"));
+    BOOST_TEST(quickbook::glob("*b*", "bc"));
+    BOOST_TEST(quickbook::glob("*b*", "abc"));
+    BOOST_TEST(!quickbook::glob("*b*", ""));
+    BOOST_TEST(!quickbook::glob("*b*", "a"));
+    BOOST_TEST(!quickbook::glob("*b*", "ac"));
+
+    BOOST_TEST(quickbook::glob("hello.txt", "hello.txt"));
+    BOOST_TEST(!quickbook::glob("world.txt", "helloworld.txt"));
+    BOOST_TEST(quickbook::glob("*world.txt", "helloworld.txt"));
+    BOOST_TEST(!quickbook::glob("world.txt*", "helloworld.txt"));
+    BOOST_TEST(!quickbook::glob("hello", "helloworld.txt"));
+    BOOST_TEST(!quickbook::glob("*hello", "helloworld.txt"));
+    BOOST_TEST(quickbook::glob("hello*", "helloworld.txt"));
+    BOOST_TEST(quickbook::glob("*world*", "helloworld.txt"));
+
+    BOOST_TEST(quickbook::glob("?", "a"));
+    BOOST_TEST(!quickbook::glob("?", ""));
+    BOOST_TEST(!quickbook::glob("?", "ab"));
+    BOOST_TEST(quickbook::glob("a?", "ab"));
+    BOOST_TEST(quickbook::glob("?b", "ab"));
+    BOOST_TEST(quickbook::glob("?bc", "abc"));
+    BOOST_TEST(quickbook::glob("a?c", "abc"));
+    BOOST_TEST(quickbook::glob("ab?", "abc"));
+    BOOST_TEST(!quickbook::glob("?bc", "aac"));
+    BOOST_TEST(!quickbook::glob("a?c", "bbc"));
+    BOOST_TEST(!quickbook::glob("ab?", "abcd"));
+
+    BOOST_TEST(quickbook::glob("[a]", "a"));
+    BOOST_TEST(!quickbook::glob("[^a]", "a"));
+    BOOST_TEST(!quickbook::glob("[b]", "a"));
+    BOOST_TEST(quickbook::glob("[^b]", "a"));
+    BOOST_TEST(quickbook::glob("[a-z]", "a"));
+    BOOST_TEST(!quickbook::glob("[^a-z]", "a"));
+    BOOST_TEST(!quickbook::glob("[b-z]", "a"));
+    BOOST_TEST(quickbook::glob("[^b-z]", "a"));
+    BOOST_TEST(quickbook::glob("[-a]", "a"));
+    BOOST_TEST(quickbook::glob("[-a]", "-"));
+    BOOST_TEST(!quickbook::glob("[-a]", "b"));
+    BOOST_TEST(!quickbook::glob("[^-a]", "a"));
+    BOOST_TEST(!quickbook::glob("[^-a]", "-"));
+    BOOST_TEST(quickbook::glob("[^-a]", "b"));
+    BOOST_TEST(quickbook::glob("[a-]", "a"));
+    BOOST_TEST(quickbook::glob("[a-]", "-"));
+    BOOST_TEST(!quickbook::glob("[a-]", "b"));
+    BOOST_TEST(!quickbook::glob("[^a-]", "a"));
+    BOOST_TEST(!quickbook::glob("[^a-]", "-"));
+    BOOST_TEST(quickbook::glob("[^a-]", "b"));
+    BOOST_TEST(quickbook::glob("[a-ce-f]", "a"));
+    BOOST_TEST(!quickbook::glob("[a-ce-f]", "d"));
+    BOOST_TEST(quickbook::glob("[a-ce-f]", "f"));
+    BOOST_TEST(!quickbook::glob("[a-ce-f]", "g"));
+    BOOST_TEST(!quickbook::glob("[^a-ce-f]", "a"));
+    BOOST_TEST(quickbook::glob("[^a-ce-f]", "d"));
+    BOOST_TEST(!quickbook::glob("[^a-ce-f]", "f"));
+    BOOST_TEST(quickbook::glob("[^a-ce-f]", "g"));
+    BOOST_TEST(!quickbook::glob("[b]", "a"));
+    BOOST_TEST(quickbook::glob("[a]bc", "abc"));
+    BOOST_TEST(quickbook::glob("a[b]c", "abc"));
+    BOOST_TEST(quickbook::glob("ab[c]", "abc"));
+    BOOST_TEST(quickbook::glob("a[a-c]c", "abc"));
+    BOOST_TEST(quickbook::glob("*[b]*", "abc"));
+    BOOST_TEST(quickbook::glob("[\\]]", "]"));
+    BOOST_TEST(!quickbook::glob("[^\\]]", "]"));
+
+    BOOST_TEST(quickbook::glob("b*ana", "banana"));
+    BOOST_TEST(quickbook::glob("1234*1234*1234", "123412341234"));
+    BOOST_TEST(!quickbook::glob("1234*1234*1234", "1234123341234"));
+    BOOST_TEST(quickbook::glob("1234*1234*1234", "123412312312341231231234"));
+    BOOST_TEST(!quickbook::glob("1234*1234*1234", "12341231231234123123123"));
+}
+
+int main()
+{
+    glob_tests();
+    return boost::report_errors();
+}