[Boost-commit] svn:boost r52990 - in trunk/boost/spirit/home: lex/lexer lex/lexer/lexertl qi/detail

Subject: [Boost-commit] svn:boost r52990 - in trunk/boost/spirit/home: lex/lexer lex/lexer/lexertl qi/detail
From: hartmut.kaiser_at_[hidden]
Date: 2009-05-14 09:17:55

Author: hkaiser
Date: 2009-05-14 09:17:53 EDT (Thu, 14 May 2009)
New Revision: 52990
URL: http://svn.boost.org/trac/boost/changeset/52990

Log:
Spirit: Added assertions ensuring a lexer token_def is associated with exactly one lexer state
Text files modified: 
   trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp |    23 +++++++++++++++++++++--                 
   trunk/boost/spirit/home/lex/lexer/token_def.hpp     |    15 ++++++++++++---                         
   trunk/boost/spirit/home/lex/lexer/token_set.hpp     |    11 ++++++++++-                             
   trunk/boost/spirit/home/qi/detail/construct.hpp     |     6 ------                                  
   4 files changed, 43 insertions(+), 12 deletions(-)

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp	(original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp	2009-05-14 09:17:53 EDT (Thu, 14 May 2009)
@@ -257,20 +257,24 @@
 
     public:
         // interface for token definition management
-        void add_token (char_type const* state, char_type tokendef, 
+        void add_token(char_type const* state, char_type tokendef, 
             std::size_t token_id)
         {
             add_state(state);
             rules.add(state, detail::escape(tokendef), token_id, state);
             initialized_dfa = false;
         }
-        void add_token (char_type const* state, string_type const& tokendef, 
+        void add_token(char_type const* state, string_type const& tokendef, 
             std::size_t token_id)
         {
             add_state(state);
             rules.add(state, tokendef, token_id, state);
             initialized_dfa = false;
         }
+
+        // Allow a token_set to be associated with this lexer instance. This 
+        // copies all token definitions of the right hand side into this lexer
+        // instance.
         void add_token(char_type const* state, token_set const& tokset)
         {
             add_state(state);
@@ -278,6 +282,19 @@
             initialized_dfa = false;
         }
 
+        // Allow to associate a whole lexer instance with another lexer 
+        // instance. This copies all token definitions of the right hand side 
+        // lexer into this instance.
+        template <typename Token_, typename Iterator_, typename Functor_
+          , typename TokenSet_>
+        void add_token(char_type const* state
+          , lexer<Token_, Iterator_, Functor_, TokenSet_> const& lexer_def)
+        {
+            add_state(state);
+            rules.add(state, lexer_def.get_rules());
+            initialized_dfa = false;
+        }
+
         // interface for pattern definition management
         void add_pattern (char_type const* state, string_type const& name,
             string_type const& patterndef)
@@ -287,6 +304,8 @@
             initialized_dfa = false;
         }
 
+        boost::lexer::rules const& get_rules() const { return rules; }
+
         void clear(char_type const* state)
         {
             std::size_t s = rules.state(state);

Modified: trunk/boost/spirit/home/lex/lexer/token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_def.hpp	(original)
+++ trunk/boost/spirit/home/lex/lexer/token_def.hpp	2009-05-14 09:17:53 EDT (Thu, 14 May 2009)
@@ -149,14 +149,23 @@
         template <typename LexerDef, typename String>
         void collect(LexerDef& lexdef, String const& state) const
         {
-            token_state = lexdef.add_state(state.c_str());
+            std::size_t state_id = lexdef.add_state(state.c_str());
+
+            // If the following assertion fires you are probably trying to use 
+            // a single token_def instance in more than one lexer state. This 
+            // is not possible. Please create a separate token_def instance 
+            // from the same regular expression for each lexer state it needs 
+            // to be associated with.
+            BOOST_ASSERT(~0 == token_state || state_id == token_state);
+
+            token_state = state_id;
             if (0 == token_id)
                 token_id = next_id<Idtype>::get();
 
             if (0 == def.which())
-                lexdef.add_token (state.c_str(), get<string_type>(def), token_id);
+                lexdef.add_token(state.c_str(), get<string_type>(def), token_id);
             else
-                lexdef.add_token (state.c_str(), get<char_type>(def), token_id);
+                lexdef.add_token(state.c_str(), get<char_type>(def), token_id);
         }
 
     public:

Modified: trunk/boost/spirit/home/lex/lexer/token_set.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_set.hpp	(original)
+++ trunk/boost/spirit/home/lex/lexer/token_set.hpp	2009-05-14 09:17:53 EDT (Thu, 14 May 2009)
@@ -116,7 +116,16 @@
         template <typename LexerDef, typename String>
         void collect(LexerDef& lexdef, String const& state) const
         {
-            token_state = lexdef.add_state(state.c_str());
+            std::size_t state_id = lexdef.add_state(state.c_str());
+
+            // If the following assertion fires you are probably trying to use 
+            // a single token_set instance in more than one lexer state. This 
+            // is not possible. Please create a separate token_set instance 
+            // from the same set of regular expressions for each lexer state it 
+            // needs to be associated with.
+            BOOST_ASSERT(~0 == token_state || state_id == token_state);
+
+            token_state = state_id;
             lexdef.add_token (state.c_str(), *this);
         }
 

Modified: trunk/boost/spirit/home/qi/detail/construct.hpp
==============================================================================
--- trunk/boost/spirit/home/qi/detail/construct.hpp	(original)
+++ trunk/boost/spirit/home/qi/detail/construct.hpp	2009-05-14 09:17:53 EDT (Thu, 14 May 2009)
@@ -15,12 +15,6 @@
 #include <boost/spirit/home/qi/parse.hpp>
 #include <boost/spirit/home/support/common_terminals.hpp>
 
-namespace boost { namespace spirit { namespace qi
-{
-    template <typename Iterator, typename Expr, typename Attr>
-    bool parse(Iterator& first, Iterator last, Expr const& expr, Attr& attr);
-}}}
-
 namespace boost { namespace spirit { namespace qi { namespace detail
 {
     namespace construct_