[Boost-commit] svn:boost r66777 - in trunk/libs/spirit: doc/lex example/lex test/karma

Subject: [Boost-commit] svn:boost r66777 - in trunk/libs/spirit: doc/lex example/lex test/karma
From: hartmut.kaiser_at_[hidden]
Date: 2010-11-26 13:33:33

Author: hkaiser
Date: 2010-11-26 13:33:30 EST (Fri, 26 Nov 2010)
New Revision: 66777
URL: http://svn.boost.org/trac/boost/changeset/66777

Log:
Spirit: added two more lexer examples
Added:
   trunk/libs/spirit/example/lex/lexer_debug_support.cpp   (contents, props changed)
   trunk/libs/spirit/example/lex/print_number_tokenids.cpp   (contents, props changed)
Text files modified: 
   trunk/libs/spirit/doc/lex/lexer_primitives.qbk |     2 +-                                      
   trunk/libs/spirit/example/lex/Jamfile          |     2 ++                                      
   trunk/libs/spirit/test/karma/bool.cpp          |     2 +-                                      
   3 files changed, 4 insertions(+), 2 deletions(-)

Modified: trunk/libs/spirit/doc/lex/lexer_primitives.qbk
==============================================================================
--- trunk/libs/spirit/doc/lex/lexer_primitives.qbk	(original)
+++ trunk/libs/spirit/doc/lex/lexer_primitives.qbk	2010-11-26 13:33:30 EST (Fri, 26 Nov 2010)
@@ -11,6 +11,6 @@
 [/ Describe the primitive lexer constructs, such as token_def, lexer ]
 
 [/ Describe the primitive lexer constructs usable in parsers, such as 
-   in_state[], set_state(), token(), etc. ]
+   in_state[], set_state(), token(), tokenid(), etc. ]
 
 [endsect]

Modified: trunk/libs/spirit/example/lex/Jamfile
==============================================================================
--- trunk/libs/spirit/example/lex/Jamfile	(original)
+++ trunk/libs/spirit/example/lex/Jamfile	2010-11-26 13:33:30 EST (Fri, 26 Nov 2010)
@@ -21,6 +21,7 @@
 exe example5                : example5.cpp ;
 exe example6                : example6.cpp ;
 exe print_numbers           : print_numbers.cpp ;
+exe print_number_tokenids   : print_number_tokenids.cpp ;
 exe word_count              : word_count.cpp ;
 exe word_count_functor      : word_count_functor.cpp ;
 exe word_count_lexer        : word_count_lexer.cpp ;
@@ -29,3 +30,4 @@
 exe strip_comments_lexer    : strip_comments_lexer.cpp ;
 exe custom_token_attribute  : custom_token_attribute.cpp ;
 
+exe lexer_debug_support     : lexer_debug_support.cpp ;

Added: trunk/libs/spirit/example/lex/lexer_debug_support.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/example/lex/lexer_debug_support.cpp	2010-11-26 13:33:30 EST (Fri, 26 Nov 2010)
@@ -0,0 +1,109 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+// #define BOOST_SPIRIT_LEXERTL_DEBUG 1
+
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/qi.hpp>
+#include <boost/spirit/include/phoenix.hpp>
+
+namespace lex = boost::spirit::lex;
+namespace qi = boost::spirit::qi;
+namespace phoenix = boost::phoenix;
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct language_tokens : lex::lexer<Lexer>
+{
+    language_tokens()
+    {
+        tok_float = "float";
+        tok_int = "int";
+        floatlit = "[0-9]+\\.[0-9]*";
+        intlit = "[0-9]+";
+        ws = "[ \t\n]+";
+        identifier = "[a-zA-Z_][a-zA-Z_0-9]*";
+      
+        this->self = ws [lex::_pass = lex::pass_flags::pass_ignore];
+        this->self += tok_float | tok_int | floatlit | intlit | identifier;
+        this->self += lex::char_('=');
+    }
+
+    lex::token_def<> tok_float, tok_int;
+    lex::token_def<> ws;
+    lex::token_def<double> floatlit;
+    lex::token_def<int> intlit;
+    lex::token_def<> identifier;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Iterator>
+struct language_grammar : qi::grammar<Iterator>
+{
+    template <typename Lexer>
+    language_grammar(language_tokens<Lexer> const& tok) 
+      : language_grammar::base_type(declarations)
+    {
+        declarations = +number;
+        number = 
+                tok.tok_float >> tok.identifier >> '=' >> tok.floatlit
+            |   tok.tok_int >> tok.identifier >> '=' >> tok.intlit
+            ;
+
+        declarations.name("declarations");
+        number.name("number");
+        debug(declarations);
+        debug(number);
+    }
+
+    qi::rule<Iterator> declarations;
+    qi::rule<Iterator> number;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main(int argc, char* argv[])
+{
+    // iterator type used to expose the underlying input stream
+    typedef std::string::iterator base_iterator_type;
+
+    // lexer type
+    typedef lex::lexertl::actor_lexer<
+        lex::lexertl::token<
+            base_iterator_type, boost::mpl::vector2<double, int> 
+        > > lexer_type;
+
+    // iterator type exposed by the lexer 
+    typedef language_tokens<lexer_type>::iterator_type iterator_type;
+
+    // now we use the types defined above to create the lexer and grammar
+    // object instances needed to invoke the parsing process
+    language_tokens<lexer_type> tokenizer;           // Our lexer
+    language_grammar<iterator_type> g (tokenizer);   // Our parser 
+
+    // Parsing is done based on the the token stream, not the character 
+    // stream read from the input.
+    std::string str ("float f = 3.4\nint i = 6\n");
+    base_iterator_type first = str.begin();
+
+    bool r = lex::tokenize_and_parse(first, str.end(), tokenizer, g);
+
+    if (r) {
+        std::cout << "-------------------------\n";
+        std::cout << "Parsing succeeded\n";
+        std::cout << "-------------------------\n";
+    }
+    else {
+        std::string rest(first, str.end());
+        std::cout << "-------------------------\n";
+        std::cout << "Parsing failed\n";
+        std::cout << "stopped at: \"" << rest << "\"\n";
+        std::cout << "-------------------------\n";
+    }
+
+    std::cout << "Bye... :-) \n\n";
+    return 0;
+}

Added: trunk/libs/spirit/example/lex/print_number_tokenids.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/example/lex/print_number_tokenids.cpp	2010-11-26 13:33:30 EST (Fri, 26 Nov 2010)
@@ -0,0 +1,121 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+//  This example is the equivalent to the following lex program:
+//
+//     %{
+//     #include <stdio.h>
+//     %}
+//     %%
+//     [0-9]+   { printf("%s\n", yytext); }
+//     .|\n     ;
+//     %%
+//     main()
+//     {
+//             yylex();
+//     }
+//
+//  Its purpose is to print all the (integer) numbers found in a file
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/spirit/include/qi.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+
+#include <iostream>
+#include <string>
+
+#include "example.hpp"
+
+using namespace boost::spirit;
+
+///////////////////////////////////////////////////////////////////////////////
+//  Token definition: We use the lexertl based lexer engine as the underlying 
+//                    lexer type.
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct print_numbers_tokenids : lex::lexer<Lexer>
+{
+    // define tokens and associate it with the lexer, we set the lexer flags
+    // not to match newlines while matching a dot, so we need to add the
+    // '\n' explicitly below
+    print_numbers_tokenids()
+      : print_numbers_tokenids::base_type(lex::match_flags::match_not_dot_newline)
+    {
+        this->self = lex::token_def<int>("[0-9]*") | ".|\n";
+    }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+//  Grammar definition
+///////////////////////////////////////////////////////////////////////////////
+template <typename Iterator>
+struct print_numbers_grammar : qi::grammar<Iterator>
+{
+    print_numbers_grammar()
+      : print_numbers_grammar::base_type(start)
+    {
+        // we just know, that the token ids get assigned starting min_token_id
+        // so, "[0-9]*" gets the id 'min_token_id' and ".|\n" gets the id
+        // 'min_token_id+1'.
+
+        // this prints the token ids of the matched tokens
+        start =  *(   qi::tokenid(lex::min_token_id) 
+                  |   qi::tokenid(lex::min_token_id+1)
+                  ) 
+                  [ std::cout << _1  << "\n" ]
+              ;
+    }
+
+    qi::rule<Iterator> start;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main(int argc, char* argv[])
+{
+    // iterator type used to expose the underlying input stream
+    typedef std::string::iterator base_iterator_type;
+
+    // the token type to be used, 'int' is available as the type of the token 
+    // attribute and no lexer state is supported
+    typedef lex::lexertl::token<base_iterator_type, boost::mpl::vector<int>
+      , boost::mpl::false_> token_type;
+
+    // lexer type
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+
+    // iterator type exposed by the lexer 
+    typedef print_numbers_tokenids<lexer_type>::iterator_type iterator_type;
+
+    // now we use the types defined above to create the lexer and grammar
+    // object instances needed to invoke the parsing process
+    print_numbers_tokenids<lexer_type> print_tokens;  // Our lexer
+    print_numbers_grammar<iterator_type> print;       // Our parser 
+
+    // Parsing is done based on the the token stream, not the character 
+    // stream read from the input.
+    std::string str (read_from_file(1 == argc ? "print_numbers.input" : argv[1]));
+    base_iterator_type first = str.begin();
+    bool r = lex::tokenize_and_parse(first, str.end(), print_tokens, print);
+
+    if (r) {
+        std::cout << "-------------------------\n";
+        std::cout << "Parsing succeeded\n";
+        std::cout << "-------------------------\n";
+    }
+    else {
+        std::string rest(first, str.end());
+        std::cout << "-------------------------\n";
+        std::cout << "Parsing failed\n";
+        std::cout << "stopped at: \"" << rest << "\"\n";
+        std::cout << "-------------------------\n";
+    }
+
+    std::cout << "Bye... :-) \n\n";
+    return 0;
+}
+
+
+

Modified: trunk/libs/spirit/test/karma/bool.cpp
==============================================================================
--- trunk/libs/spirit/test/karma/bool.cpp	(original)
+++ trunk/libs/spirit/test/karma/bool.cpp	2010-11-26 13:33:30 EST (Fri, 26 Nov 2010)
@@ -47,7 +47,7 @@
 
     // we need to provide (safe) convertibility to bool
 private:
-    struct dummy { void true_() {}; };
+    struct dummy { void true_() {} };
     typedef void (dummy::*safe_bool)();
 
 public: