[Boost-commit] svn:boost r66796 - in trunk/libs/spirit: doc doc/lex test test/lex

Subject: [Boost-commit] svn:boost r66796 - in trunk/libs/spirit: doc doc/lex test test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2010-11-27 11:05:16

Author: hkaiser
Date: 2010-11-27 11:05:13 EST (Sat, 27 Nov 2010)
New Revision: 66796
URL: http://svn.boost.org/trac/boost/changeset/66796

Log:
Spirit: adding more tests, updating docs
Added:
   trunk/libs/spirit/test/lex/auto_switch_lexerstate.cpp   (contents, props changed)
Text files modified: 
   trunk/libs/spirit/doc/lex/concepts.qbk       |    11 +++++++----                             
   trunk/libs/spirit/doc/what_s_new.qbk         |    20 ++++++++++++++++++++                    
   trunk/libs/spirit/test/Jamfile               |     1 +                                       
   trunk/libs/spirit/test/lex/regression009.cpp |     1 +                                       
   4 files changed, 29 insertions(+), 4 deletions(-)

Modified: trunk/libs/spirit/doc/lex/concepts.qbk
==============================================================================
--- trunk/libs/spirit/doc/lex/concepts.qbk	(original)
+++ trunk/libs/spirit/doc/lex/concepts.qbk	2010-11-27 11:05:13 EST (Sat, 27 Nov 2010)
@@ -46,14 +46,17 @@
 For any Lexer the following expressions must be valid:
 
 [table
-    [[Expression]              [Semantics]                       [Return type]]
-    [[`l.collect(def, state)`] [Add all token definitions provided 
+    [[Expression]              [Semantics]                        [Return type]]
+    [[`l.collect(def, state, targetstate)`] 
+                               [Add all token definitions provided 
                                 by this Lexer instance to the lexer
                                 state `state` of the token definition 
-                                container `def`.]                [`void`]]
+                                container `def`. After matching this token, the 
+                                lexer should be switched into the state 
+                                `targetstate` (optional)]         [`void`]]
     [[`l.add_actions(def)`]    [Add all semantic actions provided
                                 by this Lexer instance to the token
-                                definition container `def`.]     [`void`]]
+                                definition container `def`.]      [`void`]]
 ]
 
 [heading Type Expressions]

Modified: trunk/libs/spirit/doc/what_s_new.qbk
==============================================================================
--- trunk/libs/spirit/doc/what_s_new.qbk	(original)
+++ trunk/libs/spirit/doc/what_s_new.qbk	2010-11-27 11:05:13 EST (Sat, 27 Nov 2010)
@@ -45,6 +45,26 @@
    iterator_range of the matched input, as qi::token() does).
  * Added additional template parameter to the default `lexertl::token<>`
    definition: the type of the token id. This type defaults to `std::size_t`.
+ * It's now possible to attach lexer semantic actions to token definitions 
+   based on `lex::char()` and `lex::string()`.
+ * It's now possible to specify a lexer state the lexer should automatically be
+   switched to after matching certain tokens. For this reason the token 
+   definition syntax has been extended:
+``
+    template <typename Lexer>
+    struct lexer : lex::lexer<Lexer>
+    {
+        lexer()
+        {
+            int_ = "[1-9][0-9]*";
+            this->self("INITIAL", "TARGETSTATE") = int_;
+        }
+        lex::token_def<int> int_;
+    };
+``
+   This example lexer will match a `int_` token and will switch the lexer to 
+   the state `"TARGETSTATE"` afterwards. If the second argument is not 
+   specified the lexer remains in the previous state (as before).
 
 [endsect] [/ spirit_2_4_2]
 

Modified: trunk/libs/spirit/test/Jamfile
==============================================================================
--- trunk/libs/spirit/test/Jamfile	(original)
+++ trunk/libs/spirit/test/Jamfile	2010-11-27 11:05:13 EST (Sat, 27 Nov 2010)
@@ -135,6 +135,7 @@
     [ run lex/dedent_handling_phoenix.cpp   : : : : ]
     [ run lex/set_token_value_phoenix.cpp   : : : : ]
     [ run lex/semantic_actions.cpp          : : : : ]
+    [ run lex/auto_switch_lexerstate.cpp    : : : : ]
 
     # run Lex regression tests
     [ run lex/regression001.cpp             : : : : lex_regression001 ]

Added: trunk/libs/spirit/test/lex/auto_switch_lexerstate.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/test/lex/auto_switch_lexerstate.cpp	2010-11-27 11:05:13 EST (Sat, 27 Nov 2010)
@@ -0,0 +1,98 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2010 Mathias Gaunard
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+// This test makes sure that the BOL state (begin of line) is properly reset
+// if a token matched at the beginning of a line is discarded using 
+// lex::pass_fail.
+// Additionally this test makes sure the syntax 'self("state", "targetstate")'
+// works properly.
+
+// #define BOOST_SPIRIT_LEXERTL_DEBUG 1
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+#include <boost/spirit/home/phoenix/core.hpp>
+#include <boost/spirit/home/phoenix/operator.hpp>
+#include <boost/spirit/home/phoenix/statement.hpp>
+#include <boost/spirit/home/phoenix/object.hpp>
+#include <boost/spirit/home/phoenix/stl.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+namespace phoenix = boost::phoenix;
+
+typedef spirit::classic::position_iterator2<
+    spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+inline file_iterator 
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+    return file_iterator(
+        spirit::make_default_multi_pass(
+            std::istreambuf_iterator<char>(input)),
+        spirit::multi_pass<std::istreambuf_iterator<char> >(),
+        filename);
+}
+
+typedef lex::lexertl::token<file_iterator> token_type;
+
+struct lexer
+  : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+    lexer() : word("^[a-zA-Z0-9]+$", 1)
+    {
+        self("INITIAL", "O") = 
+                word 
+            |   lex::string("!.*$") [ 
+                    lex::_pass = lex::pass_flags::pass_ignore 
+                ]
+            |   lex::token_def<>('\n', 2) 
+            ;
+        
+        self("O", "INITIAL") = 
+                lex::string(".") [ 
+                    lex::_pass = lex::pass_flags::pass_fail 
+                ]
+            ;
+    }
+    
+    lex::token_def<> word;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+    std::stringstream ss;
+    ss << "!foo\nbar\n!baz";
+    
+    file_iterator begin = make_file_iterator(ss, "SS");
+    file_iterator end;
+    
+    lexer l;
+    token_iterator begin2 = l.begin(begin, end);
+    token_iterator end2 = l.end();
+    
+    int test_data[] = { 2, 1, 2 };
+    std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+    token_iterator it = begin2;
+    int i = 0;
+    for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+    {
+        BOOST_TEST(it->id() == test_data[i]);
+    }
+    BOOST_TEST(it == end2);
+    BOOST_TEST(i == test_data_size);
+
+    return boost::report_errors();
+}

Modified: trunk/libs/spirit/test/lex/regression009.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/regression009.cpp	(original)
+++ trunk/libs/spirit/test/lex/regression009.cpp	2010-11-27 11:05:13 EST (Sat, 27 Nov 2010)
@@ -5,6 +5,7 @@
 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
 #define BOOST_SPIRIT_DEBUG 1    // required for token streaming
+#define BOOST_SPIRIT_LEXERTL_DEBUG 1
 
 #include <boost/config/warning_disable.hpp>
 #include <boost/detail/lightweight_test.hpp>