[Boost-commit] svn:boost r54690 - in trunk/libs/spirit: doc/lex test/lex

Subject: [Boost-commit] svn:boost r54690 - in trunk/libs/spirit: doc/lex test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2009-07-05 17:29:24

Author: hkaiser
Date: 2009-07-05 17:29:23 EDT (Sun, 05 Jul 2009)
New Revision: 54690
URL: http://svn.boost.org/trac/boost/changeset/54690

Log:
Spirit: More work on lexer placeholder _value
Text files modified: 
   trunk/libs/spirit/doc/lex/lexer_primitives.qbk       |     2 +-                                      
   trunk/libs/spirit/doc/lex/lexer_semantic_actions.qbk |     3 +++                                     
   trunk/libs/spirit/test/lex/set_token_value.cpp       |    35 +++++++++++++++++++++++++----------     
   3 files changed, 29 insertions(+), 11 deletions(-)

Modified: trunk/libs/spirit/doc/lex/lexer_primitives.qbk
==============================================================================
--- trunk/libs/spirit/doc/lex/lexer_primitives.qbk	(original)
+++ trunk/libs/spirit/doc/lex/lexer_primitives.qbk	2009-07-05 17:29:23 EDT (Sun, 05 Jul 2009)
@@ -8,7 +8,7 @@
 
 [section:lexer_primitives Lexer Primitives]
 
-[/ Describe the primitive lexer constructs, such as token_def, token_set? ]
+[/ Describe the primitive lexer constructs, such as token_def, lexer ]
 
 [/ Describe the primitive lexer constructs usable in parsers, such as 
    in_state[], set_state(), token(), etc. ]

Modified: trunk/libs/spirit/doc/lex/lexer_semantic_actions.qbk
==============================================================================
--- trunk/libs/spirit/doc/lex/lexer_semantic_actions.qbk	(original)
+++ trunk/libs/spirit/doc/lex/lexer_semantic_actions.qbk	2009-07-05 17:29:23 EDT (Sun, 05 Jul 2009)
@@ -132,6 +132,9 @@
     [[`_tokenid`]
      [Refers to the token id of the token to be generated. Any modifications 
       to this value will be reflected in the generated token.]]
+    [[`_value`]
+     [Refers to the value the next token will be initialized from. Any 
+      modifications to this value will be reflected in the generated token.]]
     [[`_state`]
      [Refers to the lexer state the input has been match in. Any modifications 
       to this value will be reflected in the lexer itself (the next match will 

Modified: trunk/libs/spirit/test/lex/set_token_value.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/set_token_value.cpp	(original)
+++ trunk/libs/spirit/test/lex/set_token_value.cpp	2009-07-05 17:29:23 EDT (Sun, 05 Jul 2009)
@@ -3,7 +3,7 @@
 //  Distributed under the Boost Software License, Version 1.0. (See accompanying 
 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
-#define BOOST_SPIRIT_LEXERTL_DEBUG
+// #define BOOST_SPIRIT_LEXERTL_DEBUG
 
 #include <boost/detail/lightweight_test.hpp>
 #include <boost/spirit/include/phoenix_object.hpp>
@@ -37,12 +37,10 @@
         if (is_indent(start, end, level)) {
             id = ID_INDENT;
             ctx.set_value(level);
-            pass = lex::pass_flags::pass_use_value;
         }
         else if (is_dedent(start, end, level)) {
             id = ID_DEDENT;
             ctx.set_value(level);
-            pass = lex::pass_flags::pass_use_value;
         }
         else {
             pass = lex::pass_flags::pass_ignore;
@@ -185,7 +183,9 @@
     {
         set_token_value<lexer_type> lexer;
         std::vector<token_type> tokens;
-        std::string input("    \n        ");
+        std::string input(
+            "    \n"
+            "        \n");
         base_iterator_type first = input.begin();
 
         using phoenix::arg_names::_1;
@@ -195,7 +195,9 @@
         int i[] = { 8, 4, -1 };
         BOOST_TEST(test_indents(i, lexer.indents));
 
-        token_data d[] = { { ID_INDENT, 1 }, { ID_INDENT, 1 }, { -1, 0 } };
+        token_data d[] = { 
+            { ID_INDENT, 1 }, { ID_INDENT, 1 }
+          , { -1, 0 } };
         BOOST_TEST(test_tokens(d, tokens));
     }
 
@@ -203,7 +205,10 @@
     {
         set_token_value<lexer_type> lexer;
         std::vector<token_type> tokens;
-        std::string input("    \n        \n    \n");
+        std::string input(
+            "    \n"
+            "        \n"
+            "    \n");
         base_iterator_type first = input.begin();
 
         using phoenix::arg_names::_1;
@@ -213,7 +218,10 @@
         int i[] = { 4, -1 };
         BOOST_TEST(test_indents(i, lexer.indents));
 
-        token_data d[] = { { ID_INDENT, 1 }, { ID_INDENT, 1 }, { ID_DEDENT, 1 }, { -1, 0 } };
+        token_data d[] = { 
+            { ID_INDENT, 1 }, { ID_INDENT, 1 }
+          , { ID_DEDENT, 1 }
+          , { -1, 0 } };
         BOOST_TEST(test_tokens(d, tokens));
     }
 
@@ -221,17 +229,24 @@
     {
         set_token_value<lexer_type> lexer;
         std::vector<token_type> tokens;
-        std::string input("    \n        \n\n");
+        std::string input(
+            "    \n"
+            "        \n"
+            "            \n"
+            "    \n");
         base_iterator_type first = input.begin();
 
         using phoenix::arg_names::_1;
         BOOST_TEST(lex::tokenize(first, input.end(), lexer
           , phoenix::push_back(phoenix::ref(tokens), _1)));
 
-        int i[] = { -1 };
+        int i[] = { 4, -1 };
         BOOST_TEST(test_indents(i, lexer.indents));
 
-        token_data d[] = { { ID_INDENT, 1 }, { ID_INDENT, 1 }, { ID_DEDENT, 2 }, { -1, 0 } };
+        token_data d[] = { 
+            { ID_INDENT, 1 }, { ID_INDENT, 1 }, { ID_INDENT, 1 }
+          , { ID_DEDENT, 2 }
+          , { -1, 0 } };
         BOOST_TEST(test_tokens(d, tokens));
     }