Boost logo

Boost-Commit :

Subject: [Boost-commit] svn:boost r66741 - in trunk/libs/spirit/test: . lex
From: hartmut.kaiser_at_[hidden]
Date: 2010-11-24 18:51:58


Author: hkaiser
Date: 2010-11-24 18:51:56 EST (Wed, 24 Nov 2010)
New Revision: 66741
URL: http://svn.boost.org/trac/boost/changeset/66741

Log:
Spirit: adding new test for lexer lookahead
Added:
   trunk/libs/spirit/test/lex/regression009.cpp (contents, props changed)
Text files modified:
   trunk/libs/spirit/test/Jamfile | 1 +
   trunk/libs/spirit/test/lex/regression007.cpp | 14 +++++++++-----
   trunk/libs/spirit/test/lex/regression008.cpp | 14 +++++++++-----
   3 files changed, 19 insertions(+), 10 deletions(-)

Modified: trunk/libs/spirit/test/Jamfile
==============================================================================
--- trunk/libs/spirit/test/Jamfile (original)
+++ trunk/libs/spirit/test/Jamfile 2010-11-24 18:51:56 EST (Wed, 24 Nov 2010)
@@ -148,6 +148,7 @@
     [ run lex/regression006.cpp : : : : lex_regression006 ]
     [ run lex/regression007.cpp : : : : lex_regression007 ]
     [ run lex/regression008.cpp : : : : lex_regression008 ]
+ [ run lex/regression009.cpp : : : : lex_regression009 ]
 
     # support tests
     [ compile support/multi_pass_regression001.cpp : : support_multi_pass_regression001 ]

Modified: trunk/libs/spirit/test/lex/regression007.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/regression007.cpp (original)
+++ trunk/libs/spirit/test/lex/regression007.cpp 2010-11-24 18:51:56 EST (Wed, 24 Nov 2010)
@@ -4,8 +4,10 @@
 // Distributed under the Boost Software License, Version 1.0. (See accompanying
 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
-#include <boost/detail/lightweight_test.hpp>
+// #define BOOST_SPIRIT_LEXERTL_DEBUG
+
 #include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
 
 #include <boost/spirit/include/support_multi_pass.hpp>
 #include <boost/spirit/include/classic_position_iterator.hpp>
@@ -27,14 +29,14 @@
 
 typedef boost::iterator_range<file_iterator> file_range;
 
-inline file_iterator make_file_iterator(std::istream& input, const std::string& filename)
+inline file_iterator
+make_file_iterator(std::istream& input, const std::string& filename)
 {
     return file_iterator(
         spirit::make_default_multi_pass(
             std::istreambuf_iterator<char>(input)),
         spirit::multi_pass<std::istreambuf_iterator<char>>(),
- filename
- );
+ filename);
 }
 
 struct identifier
@@ -93,11 +95,13 @@
     int test_data[] = { 1, 3, 2 };
     std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
 
+ token_iterator it = begin2;
     int i = 0;
- for (token_iterator it = begin2; it != end2 && i < test_data_size; ++it, ++i)
+ for (/**/; it != end2 && i < test_data_size; ++it, ++i)
     {
         BOOST_TEST(it->id() == test_data[i]);
     }
+ BOOST_TEST(it == end2);
     BOOST_TEST(i == test_data_size);
 
     return boost::report_errors();

Modified: trunk/libs/spirit/test/lex/regression008.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/regression008.cpp (original)
+++ trunk/libs/spirit/test/lex/regression008.cpp 2010-11-24 18:51:56 EST (Wed, 24 Nov 2010)
@@ -4,8 +4,10 @@
 // Distributed under the Boost Software License, Version 1.0. (See accompanying
 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
-#include <boost/detail/lightweight_test.hpp>
+// #define BOOST_SPIRIT_LEXERTL_DEBUG 1
+
 #include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
 
 #include <boost/spirit/include/support_multi_pass.hpp>
 #include <boost/spirit/include/classic_position_iterator.hpp>
@@ -25,14 +27,14 @@
     spirit::multi_pass<std::istreambuf_iterator<char> >
> file_iterator;
 
-inline file_iterator make_file_iterator(std::istream& input, const std::string& filename)
+inline file_iterator
+make_file_iterator(std::istream& input, const std::string& filename)
 {
     return file_iterator(
         spirit::make_default_multi_pass(
             std::istreambuf_iterator<char>(input)),
         spirit::multi_pass<std::istreambuf_iterator<char> >(),
- filename
- );
+ filename);
 }
 
 struct identifier
@@ -100,11 +102,13 @@
     int test_data[] = { 1, 3, 2 };
     std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
 
+ token_iterator it = begin2;
     int i = 0;
- for (token_iterator it = begin2; it != end2; ++it, ++i)
+ for (/**/; it != end2 && i < test_data_size; ++it, ++i)
     {
         BOOST_TEST(it->id() == test_data[i]);
     }
+ BOOST_TEST(it == end2);
     BOOST_TEST(i == test_data_size);
 
     return boost::report_errors();

Added: trunk/libs/spirit/test/lex/regression009.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/test/lex/regression009.cpp 2010-11-24 18:51:56 EST (Wed, 24 Nov 2010)
@@ -0,0 +1,103 @@
+// Copyright (c) 2001-2010 Hartmut Kaiser
+// Copyright (c) 2010 Mathias Gaunard
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#define BOOST_SPIRIT_DEBUG 1 // required for token streaming
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+#include <boost/spirit/home/phoenix/core.hpp>
+#include <boost/spirit/home/phoenix/operator.hpp>
+#include <boost/spirit/home/phoenix/statement.hpp>
+#include <boost/spirit/home/phoenix/object.hpp>
+#include <boost/spirit/home/phoenix/stl.hpp>
+
+#include <sstream>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+namespace phoenix = boost::phoenix;
+
+typedef spirit::classic::position_iterator2<
+ spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+typedef boost::iterator_range<file_iterator> file_range;
+
+inline file_iterator
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+ return file_iterator(
+ spirit::make_default_multi_pass(
+ std::istreambuf_iterator<char>(input)),
+ spirit::multi_pass<std::istreambuf_iterator<char> >(),
+ filename);
+}
+
+struct string_literal
+{
+ string_literal(file_iterator begin, file_iterator end)
+ {
+ }
+};
+
+typedef lex::lexertl::token<
+ file_iterator, boost::mpl::vector<string_literal>
+> token_type;
+
+struct lexer
+ : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+ lexer() : st("'[^'\\n]*'", 1)
+ {
+ lex::token_def<> string_lookahead('\'');
+ self("LA") = string_lookahead;
+
+ // make sure lookahead is implicitly evaluated using the lexer state
+ // the token_def has been associated with
+ self = st [
+ phoenix::if_(lex::lookahead(string_lookahead)) [ lex::more() ]
+ ]
+ ;
+ }
+
+ lex::token_def<string_literal> st;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+ std::stringstream ss;
+ ss << "'foo''bar'";
+
+ file_iterator begin = make_file_iterator(ss, "SS");
+ file_iterator end;
+
+ lexer l;
+ token_iterator begin2 = l.begin(begin, end);
+ token_iterator end2 = l.end();
+
+ char const* test_data[] = { "1,'foo'", "1,'foo''bar'" };
+ std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+ token_iterator it = begin2;
+ int i = 0;
+ for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+ {
+ std::stringstream ss;
+ ss << it->id() << "," << *it;
+ BOOST_TEST(ss.str() == test_data[i]);
+ }
+ BOOST_TEST(it == end2);
+ BOOST_TEST(i == test_data_size);
+
+ return boost::report_errors();
+}


Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk