[Boost-commit] svn:boost r73132 - trunk/libs/spirit/example/qi/compiler_tutorial/conjure2

Subject: [Boost-commit] svn:boost r73132 - trunk/libs/spirit/example/qi/compiler_tutorial/conjure2
From: hartmut.kaiser_at_[hidden]
Date: 2011-07-15 21:55:48

Author: hkaiser
Date: 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
New Revision: 73132
URL: http://svn.boost.org/trac/boost/changeset/73132

Log:
Spirit: simplified token-id interface for lexer
Text files modified: 
   trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp  |     2 +-                                      
   trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp         |     2 ++                                      
   trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp         |    40 ++++++++++++++++++++++++++++++++++++++++
   trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp     |    28 ++++++++++++++++++++--------            
   trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp |    10 +++++-----                              
   trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp     |     8 ++------                                
   6 files changed, 70 insertions(+), 20 deletions(-)

Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp	(original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp	2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -39,7 +39,7 @@
         identifier = body.expr.identifier;
         argument_list = -(identifier % ',');
 
-        start = (token(lexer::ID_VOID_KWD) | token(lexer::ID_INT_KWD))
+        start = (l.token("void") | l.token("int"))
             >   identifier
             >   '(' > argument_list > ')'
             >   '{' > body > '}'

Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp	(original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp	2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -13,3 +13,5 @@
 
 typedef std::string::const_iterator base_iterator_type;
 template client::lexer::conjure_tokens<base_iterator_type>::conjure_tokens();
+template bool client::lexer::conjure_tokens<base_iterator_type>::add_keyword(
+    std::string const&);

Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp	(original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp	2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -17,6 +17,7 @@
 #include <boost/spirit/include/lex_static_lexertl.hpp>
 #include "conjure_static_switch_lexer.hpp"
 #endif
+#include <boost/assert.hpp>
 
 namespace client { namespace lexer 
 {
@@ -67,13 +68,52 @@
     struct conjure_tokens 
       : lex::lexer<typename detail::get_lexer_type<BaseIterator>::type>
     {
+    private:
+      // get the type of any qi::raw_token(...) and qi::token(...) constructs
+        typedef typename boost::spirit::result_of::terminal<
+            boost::spirit::tag::raw_token(tokenids)
+        >::type raw_token_spec;
+
+        typedef typename boost::spirit::result_of::terminal<
+            boost::spirit::tag::token(tokenids)
+        >::type token_spec;
+
+        typedef std::map<std::string, tokenids> keyword_map_type;
+
+    protected:
+        // add a keyword to the mapping table
+        bool add_keyword(std::string const& keyword);
+
+    public:
         typedef BaseIterator base_iterator_type;
 
         conjure_tokens();
 
+        // extract a raw_token(id) when given a registered keyword
+        raw_token_spec raw_token (std::string const& kwd) const
+        {
+            namespace qi = boost::spirit::qi;
+            qi::raw_token_type raw_token;
+
+            typename keyword_map_type::const_iterator it = keywords_.find(kwd);
+            BOOST_ASSERT(it != keywords_.end());
+            return qi::raw_token((it != keywords_.end()) ? (*it).second : ID_INVALID);
+        }
+
+        token_spec token (std::string const& kwd) const
+        {
+            namespace qi = boost::spirit::qi;
+            qi::token_type token;
+
+            typename keyword_map_type::const_iterator it = keywords_.find(kwd);
+            BOOST_ASSERT(it != keywords_.end());
+            return qi::token((it != keywords_.end()) ? (*it).second : ID_INVALID);
+        }
+
         lex::token_def<std::string> identifier;
         lex::token_def<unsigned int> uint_;
         lex::token_def<bool> bool_;
+        keyword_map_type keywords_;
     };
 }}
 

Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp	(original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp	2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -19,14 +19,12 @@
 
         this->self = uint_ | bool_;
 
-        this->self.add 
-                ("void", ID_VOID_KWD) 
-                ("int", ID_INT_KWD) 
-                ("if", ID_IF_KWD) 
-                ("else", ID_ELSE_KWD)
-                ("while", ID_WHILE_KWD)
-                ("return", ID_RETURN_KWD)
-            ;
+        add_keyword("void");
+        add_keyword("int");
+        add_keyword("if");
+        add_keyword("else");
+        add_keyword("while");
+        add_keyword("return");
 
         this->self.add 
                 ("\\|\\|", ID_OP_LOGICAL_OR)
@@ -58,5 +56,19 @@
                 ]
             ;
     }
+
+    template <typename BaseIterator>
+    bool conjure_tokens<BaseIterator>::add_keyword(std::string const& keyword)
+    {
+        // ad the token to the lexer
+        tokenids id = tokenids(this->get_next_id());
+        this->self.add(keyword, id);
+
+        // store the mapping for later retrieval 
+        std::pair<typename keyword_map_type::iterator, bool> p = 
+            keywords_.insert(typename keyword_map_type::value_type(keyword, id));
+
+        return p.second;
+    }
 }}
 

Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp	(original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp	2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -50,7 +50,7 @@
             ;
 
         variable_declaration =
-                raw_token(lexer::ID_INT_KWD) 
+                l.raw_token("int") 
             >   expr.identifier
             >  -('=' > expr)
             >   ';'
@@ -64,20 +64,20 @@
             ;
 
         if_statement =
-                raw_token(lexer::ID_IF_KWD)
+                l.raw_token("if")
             >   '('
             >   expr
             >   ')'
             >   statement_
             >
                -(
-                    raw_token(lexer::ID_ELSE_KWD)
+                    l.raw_token("else")
                 >   statement_
                 )
             ;
 
         while_statement =
-                raw_token(lexer::ID_WHILE_KWD)
+                l.raw_token("while")
             >   '('
             >   expr
             >   ')'
@@ -89,7 +89,7 @@
             ;
 
         return_statement =
-                raw_token(lexer::ID_RETURN_KWD)
+                l.raw_token("return")
             >  -expr
             >   ';'
             ;

Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp	(original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp	2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -16,6 +16,8 @@
 
     enum tokenids
     {
+        ID_INVALID = - 1,
+
         ID_OP_OPERATOR = 0x10000,
         ID_OP_BINARY = 0x20000,
         ID_OP_UNARY = 0x40000,
@@ -39,12 +41,6 @@
         ID_IDENTIFIER = ID_OP_OPERATOR + 100,
         ID_COMMENT,
         ID_WHITESPACE,
-        ID_VOID_KWD,
-        ID_INT_KWD,
-        ID_IF_KWD,
-        ID_ELSE_KWD,
-        ID_WHILE_KWD,
-        ID_RETURN_KWD,
         ID_UINT,
         ID_BOOL
     };