[Boost-commit] svn:boost r58416 - in trunk/tools/bcp: . test

Subject: [Boost-commit] svn:boost r58416 - in trunk/tools/bcp: . test
From: john_at_[hidden]
Date: 2009-12-16 12:09:18

Author: johnmaddock
Date: 2009-12-16 12:09:17 EST (Wed, 16 Dec 2009)
New Revision: 58416
URL: http://svn.boost.org/trac/boost/changeset/58416

Log:
Implement better scanning of source file dependencies - library source files are now only selected as a dependency if the included header declares something that is actually defined in the source.
Also added trivial test case.
Added:
   trunk/tools/bcp/add_dependent_lib.cpp   (contents, props changed)
   trunk/tools/bcp/test/
   trunk/tools/bcp/test/Jamfile.v2   (contents, props changed)
Text files modified: 
   trunk/tools/bcp/Jamfile.v2   |     3                                         
   trunk/tools/bcp/add_path.cpp |    95 +++++++++++++++++++-------------------- 
   trunk/tools/bcp/bcp_imp.hpp  |    10 ++--                                    
   trunk/tools/bcp/main.cpp     |     2                                         
   4 files changed, 55 insertions(+), 55 deletions(-)

Modified: trunk/tools/bcp/Jamfile.v2
==============================================================================
--- trunk/tools/bcp/Jamfile.v2	(original)
+++ trunk/tools/bcp/Jamfile.v2	2009-12-16 12:09:17 EST (Wed, 16 Dec 2009)
@@ -4,14 +4,13 @@
 
 exe bcp
     :
-    add_path.cpp bcp_imp.cpp copy_path.cpp file_types.cpp
+    add_dependent_lib.cpp add_path.cpp bcp_imp.cpp copy_path.cpp file_types.cpp
     fileview.cpp main.cpp path_operations.cpp scan_cvs_path.cpp 
     licence_info.cpp scan_licence.cpp output_licence_info.cpp
     /boost/filesystem//boost_filesystem
     /boost/regex//boost_regex
     /boost/test//boost_prg_exec_monitor
     :
-    <define>BOOST_REGEX_NO_LIB=1
     :
     release
     ;

Added: trunk/tools/bcp/add_dependent_lib.cpp
==============================================================================
--- (empty file)
+++ trunk/tools/bcp/add_dependent_lib.cpp	2009-12-16 12:09:17 EST (Wed, 16 Dec 2009)
@@ -0,0 +1,210 @@
+/*
+ *
+ * Copyright (c) 2009 Dr John Maddock
+ * Use, modification and distribution is subject to the 
+ * Boost Software License, Version 1.0. (See accompanying file 
+ * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * This file implements the following:
+ *    void bcp_implementation::add_path(const fs::path& p)
+ *    void bcp_implementation::add_directory(const fs::path& p)
+ *    void bcp_implementation::add_file(const fs::path& p)
+ *    void bcp_implementation::add_dependent_lib(const std::string& libname, const fs::path& p, const fileview& view)
+ */
+
+#include "bcp_imp.hpp"
+#include "fileview.hpp"
+#include <boost/regex.hpp>
+#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/exception.hpp>
+#include <iostream>
+
+//
+// This file contains the code required to work out whether the source/header file being scanned
+// is actually dependent upon some library's source code or not.
+//
+
+static std::map<std::string, boost::regex> scanner;
+
+static std::map<std::string, std::set<std::string> > free_function_names;
+static std::map<std::string, std::set<std::string> > class_names;
+static std::map<std::string, std::set<std::string> > variable_names;
+
+static void init_library_scanner(const fs::path& p, bool cvs_mode, const std::string& libname, bool recurse = false)
+{
+   /*
+   if(free_function_names.count(libname) == 0)
+   {
+      free_function_names[libname] = "[\\x0]";
+      class_names[libname] = "[\\x0]";
+      variable_names[libname] = "[\\x0]";
+   }
+   */
+   //
+   // Don't add files created by build system:
+   //
+   if((p.leaf() == "bin") || (p.leaf() == "bin-stage"))
+      return; 
+   //
+   // Don't add version control directories:
+   //
+   if((p.leaf() == "CVS") || (p.leaf() == ".svn"))
+      return; 
+   //
+   // don't add directories not under version control:
+   //
+   if(cvs_mode && !fs::exists(p / "CVS/Entries"))
+      return;
+   if(cvs_mode && !fs::exists(p / ".svn/entries"))
+      return;
+   //
+   // Enumerate files and directories:
+   //
+   fs::directory_iterator i(p);
+   fs::directory_iterator j;
+   while(i != j)
+   {
+      if(fs::is_directory(*i))
+         init_library_scanner(*i, cvs_mode, libname, true);
+      if(bcp_implementation::is_source_file(*i))
+      {
+         static boost::regex function_scanner(
+            "(?|"                       // Branch reset group
+               "(?:\\<\\w+\\>[^>;{},:]*)"  // Return type
+               "(?:"
+                  "(\\<\\w+\\>)"           // Maybe class name
+                  "\\s*"
+                  "(?:<[^>;{]*>)?"         // Maybe template specialisation
+                  "::\\s*)?"
+               "(\\<(?!throw|if|while|for|catch)\\w+\\>)" // function name
+               "\\s*"
+               "\\("
+                  "[^\\(\\);{}]*"          // argument list
+               "\\)"
+               "\\s*"
+               "\\{"                       // start of definition
+            "|"
+               "(\\<\\w+\\>)"              // Maybe class name
+               "\\s*"
+               "(?:<[^>;{]*>)?"            // Maybe template specialisation
+               "::\\s*"
+               "~?\\1"                     // function name, same as class name
+               "\\s*"
+               "\\("
+                  "[^\\(\\);{}]*"          // argument list
+               "\\)"
+               "\\s*"
+               "\\{"                       // start of definition
+            ")"                            // end branch reset
+            );
+         fileview view(*i);
+         boost::regex_iterator<const char*> a(view.begin(), view.end(), function_scanner);
+         boost::regex_iterator<const char*> b;
+         while(a != b)
+         {
+            if((*a)[1].matched)
+            {
+               std::string n = a->str(1);
+               class_names[libname].insert(n);
+            }
+            else
+            {
+               std::string n = a->str(2);
+               free_function_names[libname].insert(n);
+            }
+            ++a;
+         }
+      }
+      ++i;
+   }
+
+   if(recurse == false)
+   {
+      //
+      // Build the regular expressions:
+      //
+      const char* e1 = 
+         "^(?>[[:blank:]]*)(?!#)[^;{}\\r\\n]*"
+         "(?|"
+         "(?:class|struct)[^:;{}#]*"
+         "(";
+      // list of class names goes here...
+      const char* e2 = 
+         ")\\s*(?:<[^;{>]*>\\s*)?(?::[^;{]*)?\\{"
+         "|"
+         "\\<(?!return)\\w+\\>[^:;{}#=<>!~%.\\w]*(";
+         // List of function names goes here...
+      const char* e3 = 
+         ")\\s*\\([^;()]*\\)\\s*;)";
+
+      std::string class_name_list;
+      std::set<std::string>::const_iterator i = class_names[libname].begin(), j = class_names[libname].end();
+      if(i != j)
+      {
+         class_name_list = *i;
+         ++i;
+         while(i != j)
+         {
+            class_name_list += "|" + *i;
+            ++i;
+         }
+      }
+      else
+      {
+         class_name_list = "[\\x0]";
+      }
+      std::string function_name_list;
+      i = free_function_names[libname].begin();
+      j = free_function_names[libname].end();
+      if(i != j)
+      {
+         function_name_list = *i;
+         ++i;
+         while(i != j)
+         {
+            function_name_list += "|" + *i;
+            ++i;
+         }
+      }
+      else
+      {
+         function_name_list = "[\\x0]";
+      }
+
+      scanner[libname] = boost::regex(e1 + class_name_list + e2 + function_name_list + e3);
+   }
+}
+
+void bcp_implementation::add_dependent_lib(const std::string& libname, const fs::path& p, const fileview& view)
+{
+   //
+   // if the boost library libname has source associated with it
+   // then add the source to our list:
+   //
+   if(fs::exists(m_boost_path / "libs" / libname / "src"))
+   {
+      if(!m_dependencies.count(fs::path("libs") / libname / "src")) 
+      {
+         if(scanner.count(libname) == 0)
+            init_library_scanner(m_boost_path / "libs" / libname / "src", m_cvs_mode, libname);
+         boost::cmatch what;
+         if(regex_search(view.begin(), view.end(), what, scanner[libname]))
+         {
+            std::cout << "INFO: tracking source dependencies of library " << libname
+               << " due to presence of \"" << what << "\" in file " << p << std::endl;
+            //std::cout << "Full text match was: " << what << std::endl;
+            m_dependencies[fs::path("libs") / libname / "src"] = p; // set up dependency tree
+            add_path(fs::path("libs") / libname / "src");
+
+            if(fs::exists(m_boost_path / "libs" / libname / "build"))
+            {
+               if(!m_dependencies.count(fs::path("libs") / libname / "build")) 
+               {
+                  m_dependencies[fs::path("libs") / libname / "build"] = p; // set up dependency tree
+                  add_path(fs::path("libs") / libname / "build");
+               }
+            }
+         }
+      }
+   }
+}

Modified: trunk/tools/bcp/add_path.cpp
==============================================================================
--- trunk/tools/bcp/add_path.cpp	(original)
+++ trunk/tools/bcp/add_path.cpp	2009-12-16 12:09:17 EST (Wed, 16 Dec 2009)
@@ -9,7 +9,7 @@
  *    void bcp_implementation::add_path(const fs::path& p)
  *    void bcp_implementation::add_directory(const fs::path& p)
  *    void bcp_implementation::add_file(const fs::path& p)
- *    void bcp_implementation::add_dependent_lib(const std::string& libname)
+ *    void bcp_implementation::add_dependent_lib(const std::string& libname, const fs::path& p, const fileview& view)
  */
 
 #include "bcp_imp.hpp"
@@ -71,8 +71,10 @@
       if(m_boost_path.string().size())
          s.erase(0, m_boost_path.string().size() + 1);
       if(!m_dependencies.count(fs::path(s))) 
+      {
          m_dependencies[fs::path(s)] = p; // set up dependency tree
-      add_path(fs::path(s));
+         add_path(fs::path(s));
+      }
       ++i;
    }
 }
@@ -136,8 +138,10 @@
             // rather than a URL:
             fs::path dep(p.branch_path() / s);
             if(!m_dependencies.count(dep)) 
+            {
                m_dependencies[dep] = p; // set up dependency tree
-            add_path(dep);
+               add_path(dep);
+            }
          }
          ++i;
       }
@@ -178,6 +182,8 @@
       std::pair<fs::path, fs::path>("boost/mpl/map/aux_/include_preprocessed.hpp", "boost/mpl/map/aux_/preprocessed"),
       std::pair<fs::path, fs::path>("boost/mpl/list/aux_/include_preprocessed.hpp", "boost/mpl/list/aux_/preprocessed"),
       std::pair<fs::path, fs::path>("libs/graph/src/python/visitor.hpp", "libs/graph/src/python"),
+      std::pair<fs::path, fs::path>("boost/test/detail/config.hpp", "libs/test/src"),
+      std::pair<fs::path, fs::path>("boost/test/detail/config.hpp", "libs/test/build"),
    };
 
    for(unsigned int n = 0; n < (sizeof(specials)/sizeof(specials[0])); ++n)
@@ -185,33 +191,15 @@
       if(0 == compare_paths(specials[n].first, p))
       {
          if(!m_dependencies.count(specials[n].second)) 
+         {
             m_dependencies[specials[n].second] = p; // set up dependency tree
-         add_path(specials[n].second);
+            add_path(specials[n].second);
+         }
       }
    }
 
 }
 
-void bcp_implementation::add_dependent_lib(const std::string& libname, const fs::path& p)
-{
-   //
-   // if the boost library libname has source associated with it
-   // then add the source to our list:
-   //
-   if(fs::exists(m_boost_path / "libs" / libname / "src"))
-   {
-      if(!m_dependencies.count(fs::path("libs") / libname / "src")) 
-         m_dependencies[fs::path("libs") / libname / "src"] = p; // set up dependency tree
-      add_path(fs::path("libs") / libname / "src");
-      if(fs::exists(m_boost_path / "libs" / libname / "build"))
-      {
-         if(!m_dependencies.count(fs::path("libs") / libname / "build")) 
-            m_dependencies[fs::path("libs") / libname / "build"] = p; // set up dependency tree
-         add_path(fs::path("libs") / libname / "build");
-      }
-   }
-}
-
 void bcp_implementation::add_file_dependencies(const fs::path& p, bool scanfile)
 {
    static const boost::regex e(
@@ -263,14 +251,18 @@
       if(fs::exists(test_file) && !fs::is_directory(test_file) && (p.branch_path().string() != "boost"))
       {
          if(!m_dependencies.count(p.branch_path() / include_file)) 
+         {
             m_dependencies[p.branch_path() / include_file] = p;
-         add_path(p.branch_path() / include_file);
+            add_path(p.branch_path() / include_file);
+         }
       }
       else if(fs::exists(m_boost_path / include_file))
       {
          if(!m_dependencies.count(include_file)) 
+         {
             m_dependencies[include_file] = p;
-         add_path(include_file);
+            add_path(include_file);
+         }
       }
       ++i;
    }
@@ -302,14 +294,18 @@
       if(fs::exists(test_file) && !fs::is_directory(test_file) && (p.branch_path().string() != "boost"))
       {
          if(!m_dependencies.count(p.branch_path() / include_file)) 
+         {
             m_dependencies[p.branch_path() / include_file] = p;
-         add_path(p.branch_path() / include_file);
+            add_path(p.branch_path() / include_file);
+         }
       }
       else if(fs::exists(m_boost_path / include_file))
       {
          if(!m_dependencies.count(include_file)) 
+         {
             m_dependencies[include_file] = p;
-         add_path(include_file);
+            add_path(include_file);
+         }
       }
       else
       {
@@ -389,27 +385,30 @@
    boost::cmatch what;
    if(boost::regex_search(view.begin(), view.end(), what, m))
    {
-      add_dependent_lib("test", p);
+      add_dependent_lib("test", p, view);
    }
-   //
-   // grab the name of the library to which the header belongs, 
-   // and if that library has source then add the source to our
-   // list:
-   //
-   // this regex catches boost/libname.hpp or boost/libname/whatever:
-   //
-   static const boost::regex lib1("boost/([^\\./]+)(?!detail).*");
-   boost::smatch swhat;
-   if(boost::regex_match(p.string(), swhat, lib1))
-   {
-      add_dependent_lib(swhat.str(1), p);
-   }
-   //
-   // and this one catches boost/x/y/whatever (for example numeric/ublas):
-   //
-   static const boost::regex lib2("boost/([^/]+/[^/]+)/(?!detail).*");
-   if(boost::regex_match(p.string(), swhat, lib2))
+   if(!scanfile)
    {
-      add_dependent_lib(swhat.str(1), p);
+      //
+      // grab the name of the library to which the header belongs, 
+      // and if that library has source then add the source to our
+      // list:
+      //
+      // this regex catches boost/libname.hpp or boost/libname/whatever:
+      //
+      static const boost::regex lib1("boost/([^\\./]+)(?!detail).*");
+      boost::smatch swhat;
+      if(boost::regex_match(p.string(), swhat, lib1))
+      {
+         add_dependent_lib(swhat.str(1), p, view);
+      }
+      //
+      // and this one catches boost/x/y/whatever (for example numeric/ublas):
+      //
+      static const boost::regex lib2("boost/([^/]+/[^/]+)/(?!detail).*");
+      if(boost::regex_match(p.string(), swhat, lib2))
+      {
+         add_dependent_lib(swhat.str(1), p, view);
+      }
    }
 }

Modified: trunk/tools/bcp/bcp_imp.hpp
==============================================================================
--- trunk/tools/bcp/bcp_imp.hpp	(original)
+++ trunk/tools/bcp/bcp_imp.hpp	2009-12-16 12:09:17 EST (Wed, 16 Dec 2009)
@@ -44,6 +44,8 @@
 public:
    bcp_implementation();
    ~bcp_implementation();
+   static bool is_source_file(const fs::path& p);
+   static bool is_html_file(const fs::path& p);
 private:
    //
    // the following are the overridden virtuals from the base class:
@@ -62,8 +64,9 @@
    void add_module(const char* p);
 
    virtual int run();
-private:
+
    // internal helper functions:
+   bool is_binary_file(const fs::path& p);
    void scan_cvs_path(const fs::path& p);
    void scan_svn_path(const fs::path& p);
    void add_path(const fs::path& p);
@@ -71,10 +74,7 @@
    void add_file(const fs::path& p);
    void copy_path(const fs::path& p);
    void add_file_dependencies(const fs::path& p, bool scanfile);
-   bool is_source_file(const fs::path& p);
-   bool is_html_file(const fs::path& p);
-   bool is_binary_file(const fs::path& p);
-   void add_dependent_lib(const std::string& libname, const fs::path& p);
+   void add_dependent_lib(const std::string& libname, const fs::path& p, const fileview& view);
    void create_path(const fs::path& p);
    // license code:
    void scan_license(const fs::path& p, const fileview& v);

Modified: trunk/tools/bcp/main.cpp
==============================================================================
--- trunk/tools/bcp/main.cpp	(original)
+++ trunk/tools/bcp/main.cpp	2009-12-16 12:09:17 EST (Wed, 16 Dec 2009)
@@ -64,6 +64,7 @@
    //
    if(argc < 2)
    {
+      std::cout << "Error: insufficient arguments, don't know what to do." << std::endl;
       show_usage();
       return 0;
    }
@@ -136,6 +137,7 @@
       }
       else if(argv[i][0] == '-')
       {
+         std::cout << "Error: Unknown argument " << argv[i] << std::endl;
          show_usage();
          return 1;
       }

Added: trunk/tools/bcp/test/Jamfile.v2
==============================================================================
--- (empty file)
+++ trunk/tools/bcp/test/Jamfile.v2	2009-12-16 12:09:17 EST (Wed, 16 Dec 2009)
@@ -0,0 +1,25 @@
+# (C) Copyright John Maddock 2006.
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
+
+path-constant boost-path : ../../.. ;
+
+run 
+    # sources
+    ../add_dependent_lib.cpp ../add_path.cpp ../bcp_imp.cpp ../copy_path.cpp ../file_types.cpp
+    ../fileview.cpp ../main.cpp ../path_operations.cpp ../scan_cvs_path.cpp 
+    ../licence_info.cpp ../scan_licence.cpp ../output_licence_info.cpp
+    /boost/filesystem//boost_filesystem
+    /boost/regex//boost_regex
+    /boost/test//boost_prg_exec_monitor
+  : # args
+    --boost=$(boost-path) --list boost
+  : # input-files
+  : # requirements
+    release
+  : # target-name
+    bcp-test
+  : # default-build
+    release
+    ;
+