[Boost-commit] svn:boost r54689 - in trunk/libs/spirit: doc example/lex test/lex

Subject: [Boost-commit] svn:boost r54689 - in trunk/libs/spirit: doc example/lex test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2009-07-05 17:28:27

Author: hkaiser
Date: 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
New Revision: 54689
URL: http://svn.boost.org/trac/boost/changeset/54689

Log:
Spirit.Lex: removed token_set class
Text files modified: 
   trunk/libs/spirit/doc/what_s_new.qbk       |    11 ++++++-----                             
   trunk/libs/spirit/example/lex/example3.cpp |    24 +++++++++---------------                
   trunk/libs/spirit/example/lex/example4.cpp |    33 +++++++++++++--------------------       
   trunk/libs/spirit/example/lex/example5.cpp |    27 ++++++++++-----------------             
   trunk/libs/spirit/example/lex/example6.cpp |    37 ++++++++++++++-----------------------   
   trunk/libs/spirit/test/lex/lexertl3.cpp    |    10 +++-------                              
   trunk/libs/spirit/test/lex/lexertl4.cpp    |    16 +++++++---------                        
   trunk/libs/spirit/test/lex/lexertl5.cpp    |    14 +++++---------                          
   8 files changed, 67 insertions(+), 105 deletions(-)

Modified: trunk/libs/spirit/doc/what_s_new.qbk
==============================================================================
--- trunk/libs/spirit/doc/what_s_new.qbk	(original)
+++ trunk/libs/spirit/doc/what_s_new.qbk	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -116,11 +116,12 @@
   Both take a possibly arbitrary number of attribute arguments as its last 
   parameters (well, the number of attributes is limited by the macro
   `SPIRIT_ARGUMENTS_LIMIT`, which defaults to `PHOENIX_LIMIT`).
-* The `lex::lexertl_lexer`, `lex::lexertl_token_set`, and `lex::lexertl_token`
+* The `lex::lexertl_lexer`, and `lex::lexertl_token`
   classes have been moved to the `lex::lexertl` namespace and the names have been 
-  changed to `lex::lexertl::lexer`, `lex::lexertl::token_set`, `lex::lexertl::token`
-  (the same applies to the `lex::lexert_actor_lexer`, and the `static_lexertl_*`
-  family of types).
+  changed to `lex::lexertl::lexer`, `lex::lexertl::token` (the same applies to 
+  the `lex::lexert_actor_lexer`, and the `static_lexertl_*` family of types).
+* The class `lex::lexertl_token_set` has been removed alltogether as its 
+  complete functionality is available from the lexer class.
 * The __lex__ library has been updated to use the newest version of Ben 
   Hansons __lexertl__ lexer construction library (Boost review pending).
 * The `lex::lexer<Lexer>` template constructor now takes an optional parameter
@@ -139,7 +140,7 @@
 * Lexer semantic actions now have to conform to a changed interface (see
   __sec_lex_semactions__ for details).
 * Added placeholder symbols usable from the inside of lexer semantic actions 
-  while using Phoenix: _start, _end, _eoi, _state, and _pass (see 
+  while using Phoenix: _start, _end, _eoi, _state, _value, and _pass (see 
   __sec_lex_semactions__ for more details).
 * Added (lazy) support functions usable from the inside of lexer semantic
    actions while using Phoenix: `lex::more()`, `lex::less()`, and 

Modified: trunk/libs/spirit/example/lex/example3.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example3.cpp	(original)
+++ trunk/libs/spirit/example/lex/example3.cpp	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -41,29 +41,25 @@
 template <typename Lexer>
 struct example3_tokens : lexer<Lexer>
 {
-    typedef typename Lexer::token_set token_set;
-
     example3_tokens()
     {
         // define the tokens to match
         ellipses = "\\.\\.\\.";
         number = "[0-9]+";
 
+        // associate the tokens and the token set with the lexer
+        this->self = ellipses | '(' | ')' | number;
+
         // define the whitespace to ignore (spaces, tabs, newlines and C-style 
         // comments)
-        white_space 
+        this->self("WS") 
             =   token_def<>("[ \\t\\n]+")               // whitespace
             |   "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"   // C style comments
             ;
-
-        // associate the tokens and the token set with the lexer
-        this->self = ellipses | '(' | ')' | number;
-        this->self("WS") = white_space;
     }
 
     // these tokens expose the iterator_range of the matched input sequence
     token_def<> ellipses, identifier, number;
-    token_set white_space;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -71,7 +67,7 @@
 ///////////////////////////////////////////////////////////////////////////////
 template <typename Iterator, typename Lexer>
 struct example3_grammar 
-  : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+  : grammar<Iterator, in_state_skipper<Lexer> >
 {
     template <typename TokenDef>
     example3_grammar(TokenDef const& tok)
@@ -95,8 +91,7 @@
         BOOST_SPIRIT_DEBUG_NODE(couplet);
     }
 
-    typedef typename Lexer::token_set token_set;
-    rule<Iterator, in_state_skipper<token_set> > start, couplet;
+    rule<Iterator, in_state_skipper<Lexer> > start, couplet;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -119,7 +114,7 @@
     typedef example3_tokens::iterator_type iterator_type;
 
     // this is the type of the grammar to parse
-    typedef example3_grammar<iterator_type, lexer_type> example3_grammar;
+    typedef example3_grammar<iterator_type, example3_tokens::lexer_def> example3_grammar;
 
     // now we use the types defined above to create the lexer and grammar
     // object instances needed to invoke the parsing process
@@ -136,9 +131,8 @@
 
     // Parsing is done based on the the token stream, not the character 
     // stream read from the input.
-    // Note, how we use the token_set defined above as the skip parser.
-    std::string ws("WS");
-    bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.white_space]);
+    // Note how we use the lexer defined above as the skip parser.
+    bool r = phrase_parse(iter, end, calc, in_state("WS")[tokens.self]);
 
     if (r && iter == end)
     {

Modified: trunk/libs/spirit/example/lex/example4.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example4.cpp	(original)
+++ trunk/libs/spirit/example/lex/example4.cpp	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -44,8 +44,6 @@
 template <typename Lexer>
 struct example4_tokens : lexer<Lexer>
 {
-    typedef typename Lexer::token_set token_set;
-
     example4_tokens()
     {
         // define the tokens to match
@@ -55,17 +53,16 @@
         else_ = "else";
         while_ = "while";
 
+        // associate the tokens and the token set with the lexer
+        this->self = token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
+        this->self += if_ | else_ | while_ | identifier;
+
         // define the whitespace to ignore (spaces, tabs, newlines and C-style 
         // comments)
-        white_space 
+        this->self("WS")
             =   token_def<>("[ \\t\\n]+") 
             |   "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
             ;
-
-        // associate the tokens and the token set with the lexer
-        this->self = token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
-        this->self += if_ | else_ | while_ | identifier;
-        this->self("WS") = white_space;
     }
 
 //[example4_token_def
@@ -88,9 +85,6 @@
     token_def<std::string> identifier;
     token_def<unsigned int> constant;
 //]
-
-    // token set to be used as the skip parser
-    token_set white_space;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -98,7 +92,7 @@
 ///////////////////////////////////////////////////////////////////////////////
 template <typename Iterator, typename Lexer>
 struct example4_grammar 
-  : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+  : grammar<Iterator, in_state_skipper<Lexer> >
 {
     template <typename TokenDef>
     example4_grammar(TokenDef const& tok)
@@ -150,15 +144,14 @@
             ;
     }
 
-    typedef typename Lexer::token_set token_set;
     typedef boost::variant<unsigned int, std::string> expression_type;
 
-    rule<Iterator, in_state_skipper<token_set> > program, block, statement;
-    rule<Iterator, in_state_skipper<token_set> > assignment, if_stmt;
-    rule<Iterator, in_state_skipper<token_set> > while_stmt;
+    rule<Iterator, in_state_skipper<Lexer> > program, block, statement;
+    rule<Iterator, in_state_skipper<Lexer> > assignment, if_stmt;
+    rule<Iterator, in_state_skipper<Lexer> > while_stmt;
 
     //  the expression is the only rule having a return value
-    rule<Iterator, expression_type(), in_state_skipper<token_set> >  expression;
+    rule<Iterator, expression_type(), in_state_skipper<Lexer> >  expression;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -195,7 +188,7 @@
     typedef example4_tokens::iterator_type iterator_type;
 
     // this is the type of the grammar to parse
-    typedef example4_grammar<iterator_type, lexer_type> example4_grammar;
+    typedef example4_grammar<iterator_type, example4_tokens::lexer_def> example4_grammar;
 
     // now we use the types defined above to create the lexer and grammar
     // object instances needed to invoke the parsing process
@@ -212,10 +205,10 @@
         
     // Parsing is done based on the the token stream, not the character 
     // stream read from the input.
-    // Note, how we use the token_set defined above as the skip parser. It must
+    // Note how we use the lexer defined above as the skip parser. It must
     // be explicitly wrapped inside a state directive, switching the lexer 
     // state for the duration of skipping whitespace.
-    bool r = phrase_parse(iter, end, calc, in_state("WS")[tokens.white_space]);
+    bool r = phrase_parse(iter, end, calc, in_state("WS")[tokens.self]);
 
     if (r && iter == end)
     {

Modified: trunk/libs/spirit/example/lex/example5.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example5.cpp	(original)
+++ trunk/libs/spirit/example/lex/example5.cpp	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -46,8 +46,6 @@
 template <typename Lexer>
 struct example5_base_tokens : lexer<Lexer>
 {
-    typedef typename Lexer::token_set token_set;
-
 protected:
     // this lexer is supposed to be used as a base type only
     example5_base_tokens() {}
@@ -61,17 +59,16 @@
         if_ = "if";
         while_ = "while";
 
+        // associate the tokens and the token set with the lexer
+        this->self += token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
+        this->self += if_ | while_ | identifier;
+
         // define the whitespace to ignore (spaces, tabs, newlines and C-style 
         // comments)
-        white_space 
+        this->self("WS")
             =   token_def<>("[ \\t\\n]+") 
             |   "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
             ;
-
-        // associate the tokens and the token set with the lexer
-        this->self += token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
-        this->self += if_ | while_ | identifier;
-        this->self("WS") = white_space;
     }
 
     // these tokens have no attribute
@@ -92,9 +89,6 @@
     // avoiding them being copied around.
     token_def<std::string> identifier;
     token_def<unsigned int> constant;
-
-    // token set to be used as the skip parser
-    token_set white_space;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -102,7 +96,7 @@
 ///////////////////////////////////////////////////////////////////////////////
 template <typename Iterator, typename Lexer>
 struct example5_base_grammar 
-  : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+  : grammar<Iterator, in_state_skipper<Lexer> >
 {
     template <typename TokenDef>
     example5_base_grammar(TokenDef const& tok)
@@ -152,7 +146,7 @@
             ;
     }
 
-    typedef in_state_skipper<typename Lexer::token_set> skipper_type;
+    typedef in_state_skipper<Lexer> skipper_type;
 
     rule<Iterator, skipper_type> program, block, statement;
     rule<Iterator, skipper_type> assignment, if_stmt;
@@ -170,7 +164,6 @@
 struct example5_tokens : example5_base_tokens<Lexer>
 {
     typedef example5_base_tokens<Lexer> base_type;
-    typedef typename Lexer::token_set token_set;
 
     example5_tokens()
     {
@@ -240,7 +233,7 @@
     typedef example5_tokens::iterator_type iterator_type;
 
     // this is the type of the grammar to parse
-    typedef example5_grammar<iterator_type, lexer_type> example5_grammar;
+    typedef example5_grammar<iterator_type, example5_tokens::lexer_def> example5_grammar;
 
     // now we use the types defined above to create the lexer and grammar
     // object instances needed to invoke the parsing process
@@ -257,11 +250,11 @@
 
     // Parsing is done based on the the token stream, not the character 
     // stream read from the input.
-    // Note, how we use the token_set defined above as the skip parser. It must
+    // Note how we use the lexer defined above as the skip parser. It must
     // be explicitly wrapped inside a state directive, switching the lexer 
     // state for the duration of skipping whitespace.
     std::string ws("WS");
-    bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.white_space]);
+    bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.self]);
 
     if (r && iter == end)
     {

Modified: trunk/libs/spirit/example/lex/example6.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example6.cpp	(original)
+++ trunk/libs/spirit/example/lex/example6.cpp	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -60,21 +60,12 @@
 template <typename Lexer>
 struct example6_tokens : lexer<Lexer>
 {
-    typedef typename Lexer::token_set token_set;
-
     example6_tokens()
     {
         // define the tokens to match
         identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
         constant = "[0-9]+";
 
-        // define the whitespace to ignore (spaces, tabs, newlines and C-style 
-        // comments)
-        white_space 
-            =   token_def<>("[ \\t\\n]+") 
-            |   "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
-            ;
-
         // associate the tokens and the token set with the lexer
         this->self = token_def<>('(') | ')' | '{' | '}' | '=' | ';';
 
@@ -90,8 +81,12 @@
             (identifier, ID_IDENTIFIER)
         ;
 
-        // add whitespace tokens to another lexer state (here: "WS")
-        this->self("WS") = white_space;
+        // define the whitespace to ignore (spaces, tabs, newlines and C-style 
+        // comments) and add those to another lexer state (here: "WS")
+        this->self("WS") 
+            =   token_def<>("[ \\t\\n]+") 
+            |   "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
+            ;
     }
 
     // The following two tokens have an associated attribute type, identifier 
@@ -109,9 +104,6 @@
     // avoiding them being copied around.
     token_def<std::string> identifier;
     token_def<unsigned int> constant;
-
-    // token set to be used as the skip parser
-    token_set white_space;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -119,7 +111,7 @@
 ///////////////////////////////////////////////////////////////////////////////
 template <typename Iterator, typename Lexer>
 struct example6_grammar 
-  : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+  : grammar<Iterator, in_state_skipper<Lexer> >
 {
     template <typename TokenDef>
     example6_grammar(TokenDef const& tok)
@@ -174,15 +166,14 @@
             ;
     }
 
-    typedef typename Lexer::token_set token_set;
     typedef boost::variant<unsigned int, std::string> expression_type;
 
-    rule<Iterator, in_state_skipper<token_set> > program, block, statement;
-    rule<Iterator, in_state_skipper<token_set> > assignment, if_stmt;
-    rule<Iterator, in_state_skipper<token_set> > while_stmt;
+    rule<Iterator, in_state_skipper<Lexer> > program, block, statement;
+    rule<Iterator, in_state_skipper<Lexer> > assignment, if_stmt;
+    rule<Iterator, in_state_skipper<Lexer> > while_stmt;
 
     //  the expression is the only rule having a return value
-    rule<Iterator, expression_type(), in_state_skipper<token_set> >  expression;
+    rule<Iterator, expression_type(), in_state_skipper<Lexer> >  expression;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -218,7 +209,7 @@
     typedef example6_tokens::iterator_type iterator_type;
 
     // this is the type of the grammar to parse
-    typedef example6_grammar<iterator_type, lexer_type> example6_grammar;
+    typedef example6_grammar<iterator_type, example6_tokens::lexer_def> example6_grammar;
 
     // now we use the types defined above to create the lexer and grammar
     // object instances needed to invoke the parsing process
@@ -235,11 +226,11 @@
 
     // Parsing is done based on the the token stream, not the character 
     // stream read from the input.
-    // Note, how we use the token_def defined above as the skip parser. It must
+    // Note how we use the lexer defined above as the skip parser. It must
     // be explicitly wrapped inside a state directive, switching the lexer 
     // state for the duration of skipping whitespace.
     std::string ws("WS");
-    bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.white_space]);
+    bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.self]);
 
     if (r && iter == end)
     {

Modified: trunk/libs/spirit/test/lex/lexertl3.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/lexertl3.cpp	(original)
+++ trunk/libs/spirit/test/lex/lexertl3.cpp	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -21,18 +21,17 @@
     std::size_t const CPPCOMMENT = 2;
     token_def c_comment ("\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/", CCOMMENT);
     token_def cpp_comment ("\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)", CPPCOMMENT);
-    token_def ws_tok ("[\\v\\f\\n\\r]*");
 
     typedef std::string::iterator base_iterator_type;    
     typedef lex::lexertl::token<base_iterator_type> token_type;
     typedef lex::lexertl::lexer<token_type> lexer_type;
 
     typedef lex::lexer<lexer_type> lexer_def;
-    typedef lexer_def::token_set token_set;
 
     {
         // initialize lexer
         std::string str("def");
+        token_def ws_tok ("[\\v\\f\\n\\r]*");
         lexer_def lex;
         lex.self = c_comment;
         lex.self += cpp_comment | '1' | '2' | '3' | "abc" | str;
@@ -49,15 +48,12 @@
     }
 
     {
-        // init a token set
-        token_set ws;
-        ws = token_def(' ') | '\t' | ws_tok;
-
         // initialize lexer
         lexer_def lex;
+        token_def ws_tok ("[\\v\\f\\n\\r]*");
         lex.self = c_comment;
         lex.self += cpp_comment | '1' | '2' | '3';
-        lex.self("WHITESPACE") = ws;
+        lex.self("WHITESPACE") = token_def(' ') | '\t' | ws_tok;
 
         // test lexer for two different input strings
         BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));

Modified: trunk/libs/spirit/test/lex/lexertl4.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/lexertl4.cpp	(original)
+++ trunk/libs/spirit/test/lex/lexertl4.cpp	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -32,13 +32,13 @@
     typedef lex::lexertl::lexer<token_type> lexer_type;
 
     typedef lex::lexer<lexer_type> lexer_def;
-    typedef lexer_def::token_set token_set;
 
     std::string str("def");
 
     {
         // initialize lexer
         lexer_def lex;
+        token_def ws_tok ("[\\v\\f\\n\\r]*", TOKEN_ID_WS);
         lex.self.add
             (c_comment)(cpp_comment) 
             ('1')('2')('3')
@@ -58,15 +58,10 @@
     }
 
     {
-        // init a token set
-        token_set ws;
-        ws.add
-            (' ')('\t')
-            (ws_tok)
-        ;
-
         // initialize lexer
         lexer_def lex;
+        token_def ws_tok ("[\\v\\f\\n\\r]*", TOKEN_ID_WS);
+
         lex.self.add
             (c_comment)(cpp_comment) 
             ('1')('2')('3')
@@ -74,7 +69,10 @@
             (str, TOKEN_ID_STR)
         ;
 
-        lex.self("WHITESPACE").add(ws);
+        lex.self("WHITESPACE").add
+            (' ')('\t')
+            (ws_tok)
+        ;
 
         // test lexer for different input strings
         BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));

Modified: trunk/libs/spirit/test/lex/lexertl5.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/lexertl5.cpp	(original)
+++ trunk/libs/spirit/test/lex/lexertl5.cpp	2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -29,7 +29,6 @@
     typedef lex::lexertl::lexer<token_type> lexer_type;
 
     typedef lex::lexer<lexer_type> lexer_def;
-    typedef lexer_def::token_set token_set;
 
     std::string str("def");
 
@@ -79,13 +78,7 @@
         token_def cpp_comment ("{CPPCOMMENT}", CPPCOMMENT);
         token_def ws_tok ("{WS}");
 
-        // init a token set
-        token_set ws;
-        ws.add
-            (' ')('\t')
-            (ws_tok, TOKEN_ID_WS)
-        ;
-
+        // init lexer
         lex.self.add
             (c_comment)(cpp_comment) 
             ('1')('2')('3')
@@ -93,7 +86,10 @@
             (str, TOKEN_ID_STR)
         ;
 
-        lex.self("WHITESPACE").add(ws);
+        lex.self("WHITESPACE").add
+            (' ')('\t')
+            (ws_tok, TOKEN_ID_WS)
+        ;
 
         // test lexer for different input strings
         BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));