[Boost-commit] svn:boost r69833 - in trunk: boost/spirit/home/lex/lexer libs/spirit/doc libs/spirit/test libs/spirit/test/lex

Subject: [Boost-commit] svn:boost r69833 - in trunk: boost/spirit/home/lex/lexer libs/spirit/doc libs/spirit/test libs/spirit/test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2011-03-10 17:29:44

Author: hkaiser
Date: 2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
New Revision: 69833
URL: http://svn.boost.org/trac/boost/changeset/69833

Log:
Spirit: adding possibility to specify token id with lex::char_ and lex::string
Added:
   trunk/libs/spirit/test/lex/string_token_id.cpp   (contents, props changed)
Text files modified: 
   trunk/boost/spirit/home/lex/lexer/char_token_def.hpp   |    89 ++++++++++++++++++++++++++++++++------- 
   trunk/boost/spirit/home/lex/lexer/string_token_def.hpp |    56 +++++++++++++++++++-----                
   trunk/libs/spirit/doc/what_s_new.qbk                   |     8 +++                                     
   trunk/libs/spirit/test/Jamfile                         |     1                                         
   4 files changed, 124 insertions(+), 30 deletions(-)

Modified: trunk/boost/spirit/home/lex/lexer/char_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/char_token_def.hpp	(original)
+++ trunk/boost/spirit/home/lex/lexer/char_token_def.hpp	2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -21,29 +21,42 @@
     ///////////////////////////////////////////////////////////////////////////
     // Enablers
     ///////////////////////////////////////////////////////////////////////////
+
+    // enables 'x'
     template <>
-    struct use_terminal<lex::domain, char>               // enables 'x'
+    struct use_terminal<lex::domain, char>
       : mpl::true_ {};
 
+    // enables "x"
     template <>
-    struct use_terminal<lex::domain, char[2]>            // enables "x"
+    struct use_terminal<lex::domain, char[2]>
       : mpl::true_ {};
 
+    // enables wchar_t
     template <>
-    struct use_terminal<lex::domain, wchar_t>            // enables wchar_t
+    struct use_terminal<lex::domain, wchar_t>
       : mpl::true_ {};
 
+    // enables L"x"
     template <>
-    struct use_terminal<lex::domain, wchar_t[2]>         // enables L"x"
+    struct use_terminal<lex::domain, wchar_t[2]>
       : mpl::true_ {};
 
+    // enables char_('x'), char_("x")
     template <typename CharEncoding, typename A0>
     struct use_terminal<lex::domain
       , terminal_ex<
-            tag::char_code<tag::char_, CharEncoding>     // enables char_('x'), char_("x")
-          , fusion::vector1<A0>
-        >
-    > : mpl::true_ {};
+            tag::char_code<tag::char_, CharEncoding>
+          , fusion::vector1<A0> > > 
+      : mpl::true_ {};
+
+    // enables char_('x', ID), char_("x", ID)
+    template <typename CharEncoding, typename A0, typename A1>
+    struct use_terminal<lex::domain
+      , terminal_ex<
+            tag::char_code<tag::char_, CharEncoding>
+          , fusion::vector2<A0, A1> > > 
+      : mpl::true_ {};
 }}
 
 namespace boost { namespace spirit { namespace lex
@@ -60,14 +73,16 @@
     //      represents a single character token definition
     //
     ///////////////////////////////////////////////////////////////////////////
-    template <typename CharEncoding = char_encoding::standard>
+    template <typename CharEncoding = char_encoding::standard
+      , typename IdType = std::size_t>
     struct char_token_def
-      : primitive_lexer<char_token_def<CharEncoding> >
+      : primitive_lexer<char_token_def<CharEncoding, IdType> >
     {
         typedef typename CharEncoding::char_type char_type;
 
-        char_token_def(char_type ch) 
-          : ch(ch), unique_id_(std::size_t(~0)), token_state_(std::size_t(~0)) 
+        char_token_def(char_type ch, IdType const& id) 
+          : ch(ch), id_(id), unique_id_(std::size_t(~0))
+          , token_state_(std::size_t(~0)) 
         {}
 
         template <typename LexerDef, typename String>
@@ -90,18 +105,18 @@
                 lexdef.add_state(target);
 
             token_state_ = state_id;
-            unique_id_ = lexdef.add_token (state.c_str(), ch
-              , static_cast<std::size_t>(ch), target);
+            unique_id_ = lexdef.add_token (state.c_str(), ch, id_, target);
         }
 
         template <typename LexerDef>
         void add_actions(LexerDef&) const {}
 
-        std::size_t id() const { return static_cast<std::size_t>(ch); }
+        IdType id() const { return id_; }
         std::size_t unique_id() const { return unique_id_; }
         std::size_t state() const { return token_state_; }
 
         char_type ch;
+        mutable IdType id_;
         mutable std::size_t unique_id_;
         mutable std::size_t token_state_;
     };
@@ -162,7 +177,8 @@
         template <typename Terminal>
         result_type operator()(Terminal const& term, unused_type) const
         {
-            return result_type(fusion::at_c<0>(term.args));
+            Char ch = fusion::at_c<0>(term.args);
+            return result_type(ch, ch);
         }
     };
 
@@ -180,7 +196,46 @@
         template <typename Terminal>
         result_type operator()(Terminal const& term, unused_type) const
         {
-            return result_type(fusion::at_c<0>(term.args)[0]);
+            Char ch = fusion::at_c<0>(term.args)[0];
+            return result_type(ch, ch);
+        }
+    };
+
+    // handle char_('x', ID)
+    template <typename CharEncoding, typename Modifiers, typename A0, typename A1>
+    struct make_primitive<
+        terminal_ex<
+            tag::char_code<tag::char_, CharEncoding>
+          , fusion::vector2<A0, A1>
+        >
+      , Modifiers>
+    {
+        typedef char_token_def<CharEncoding> result_type;
+
+        template <typename Terminal>
+        result_type operator()(Terminal const& term, unused_type) const
+        {
+            return result_type(
+                fusion::at_c<0>(term.args), fusion::at_c<1>(term.args));
+        }
+    };
+
+    // handle char_("x", ID)
+    template <typename CharEncoding, typename Modifiers, typename Char, typename A1>
+    struct make_primitive<
+        terminal_ex<
+            tag::char_code<tag::char_, CharEncoding>
+          , fusion::vector2<Char(&)[2], A1>   // single char strings
+        >
+      , Modifiers>
+    {
+        typedef char_token_def<CharEncoding> result_type;
+
+        template <typename Terminal>
+        result_type operator()(Terminal const& term, unused_type) const
+        {
+            return result_type(
+                fusion::at_c<0>(term.args)[0], fusion::at_c<1>(term.args));
         }
     };
 }}}  // namespace boost::spirit::lex

Modified: trunk/boost/spirit/home/lex/lexer/string_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/string_token_def.hpp	(original)
+++ trunk/boost/spirit/home/lex/lexer/string_token_def.hpp	2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -17,6 +17,7 @@
 #include <boost/spirit/home/lex/meta_compiler.hpp>
 #include <boost/type_traits/add_const.hpp>
 #include <boost/type_traits/add_reference.hpp>
+#include <boost/type_traits/is_integral.hpp>
 #include <boost/type_traits/remove_const.hpp>
 #include <boost/fusion/include/vector.hpp>
 #include <boost/fusion/include/at.hpp>
@@ -26,18 +27,28 @@
     ///////////////////////////////////////////////////////////////////////////
     // Enablers
     ///////////////////////////////////////////////////////////////////////////
+
+    // enables strings
     template <typename T>
     struct use_terminal<lex::domain, T
-      , typename enable_if<traits::is_string<T> >::type> // enables strings
+      , typename enable_if<traits::is_string<T> >::type>
       : mpl::true_ {};
 
+    // enables string(str)
     template <typename CharEncoding, typename A0>
     struct use_terminal<lex::domain
       , terminal_ex<
-            tag::char_code<tag::string, CharEncoding>   // enables string(str)
-          , fusion::vector1<A0> >
-    > : traits::is_string<A0> {};
+            tag::char_code<tag::string, CharEncoding>
+          , fusion::vector1<A0> > > 
+      : traits::is_string<A0> {};
 
+    // enables string(str, ID)
+    template <typename CharEncoding, typename A0, typename A1>
+    struct use_terminal<lex::domain
+      , terminal_ex<
+            tag::char_code<tag::string, CharEncoding>
+          , fusion::vector2<A0, A1> > > 
+      : traits::is_string<A0> {};
 }}
 
 namespace boost { namespace spirit { namespace lex
@@ -54,17 +65,18 @@
     //      represents a string based token definition
     //
     ///////////////////////////////////////////////////////////////////////////
-    template <typename String, typename CharEncoding = char_encoding::standard>
+    template <typename String, typename IdType = std::size_t
+      , typename CharEncoding = char_encoding::standard>
     struct string_token_def
-      : primitive_lexer<string_token_def<String, CharEncoding> >
+      : primitive_lexer<string_token_def<String, IdType, CharEncoding> >
     {
         typedef typename
             remove_const<typename traits::char_type_of<String>::type>::type
         char_type;
         typedef std::basic_string<char_type> string_type;
 
-        string_token_def(typename add_reference<String>::type str)
-          : str_(str), id_(std::size_t(~0)), unique_id_(std::size_t(~0))
+        string_token_def(typename add_reference<String>::type str, IdType const& id)
+          : str_(str), id_(id), unique_id_(std::size_t(~0))
           , token_state_(std::size_t(~0)) 
         {}
 
@@ -90,8 +102,8 @@
             token_state_ = state_id;
 
             typedef typename LexerDef::id_type id_type;
-            if (std::size_t(~0) == id_)
-                id_ = lexdef.get_next_id();
+            if (IdType(~0) == id_)
+                id_ = IdType(lexdef.get_next_id());
 
             unique_id_ = lexdef.add_token (state.c_str(), str_, id_, target);
         }
@@ -104,7 +116,7 @@
         std::size_t state() const { return token_state_; }
 
         string_type str_;
-        mutable std::size_t id_;
+        mutable IdType id_;
         mutable std::size_t unique_id_;
         mutable std::size_t token_state_;
     };
@@ -134,15 +146,33 @@
       , Modifiers>
     {
         typedef typename add_const<A0>::type const_string;
-        typedef string_token_def<const_string, CharEncoding> result_type;
+        typedef string_token_def<const_string, std::size_t, CharEncoding> 
+            result_type;
 
         template <typename Terminal>
         result_type operator()(Terminal const& term, unused_type) const
         {
-            return result_type(fusion::at_c<0>(term.args));
+            return result_type(fusion::at_c<0>(term.args), std::size_t(~0));
         }
     };
 
+    template <typename Modifiers, typename CharEncoding, typename A0, typename A1>
+    struct make_primitive<
+        terminal_ex<
+            tag::char_code<tag::string, CharEncoding>
+          , fusion::vector2<A0, A1> >
+      , Modifiers>
+    {
+        typedef typename add_const<A0>::type const_string;
+        typedef string_token_def<const_string, A1, CharEncoding> result_type;
+
+        template <typename Terminal>
+        result_type operator()(Terminal const& term, unused_type) const
+        {
+            return result_type(
+                fusion::at_c<0>(term.args), fusion::at_c<1>(term.args));
+        }
+    };
 }}}  // namespace boost::spirit::lex
 
 #endif 

Modified: trunk/libs/spirit/doc/what_s_new.qbk
==============================================================================
--- trunk/libs/spirit/doc/what_s_new.qbk	(original)
+++ trunk/libs/spirit/doc/what_s_new.qbk	2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -61,6 +61,12 @@
   instantiates the terminals that he needs. So, instead of writing `using qi::uint_`
   one writes instead: `qi::uint_type uint_`.
 
+[heading New Features in Lex]
+
+* Added the possibility to speciy a token id while creating a token definition 
+  using `lex::char_` and `lex::string`. Both primitives now accept a second 
+  parameter which will be interpreted as the requested token id for any token 
+  generated from this definition.
 
 [heading Making Stuff Work]
 
@@ -81,6 +87,8 @@
   in order for the new behavior to kick in. By default, the old behavior
   is still in place.
 * Alternatives now support attribute compatibility.
+* Fixed Boost ticket #5246: mmap_file_iterator Fails to initialize correctly.
+* Fixed Boost ticket #5246: mmap_file_iterator Fails to initialize correctly.
 
 [endsect]
 

Modified: trunk/libs/spirit/test/Jamfile
==============================================================================
--- trunk/libs/spirit/test/Jamfile	(original)
+++ trunk/libs/spirit/test/Jamfile	2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -218,6 +218,7 @@
      [ run lex/semantic_actions.cpp          : : : : lex_semantic_actions ]
      [ run lex/auto_switch_lexerstate.cpp    : : : : lex_auto_switch_lexerstate ]
      [ run lex/id_type_enum.cpp              : : : : lex_id_type_enum ]
+     [ run lex/string_token_id.cpp           : : : : lex_string_token_id ]
 
     ;
 

Added: trunk/libs/spirit/test/lex/string_token_id.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/test/lex/string_token_id.cpp	2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -0,0 +1,89 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+// #define BOOST_SPIRIT_LEXERTL_DEBUG
+#define BOOST_VARIANT_MINIMIZE_SIZE
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/qi.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_statement.hpp>
+#include <boost/spirit/include/phoenix_container.hpp>
+
+#include <iostream>
+#include <string>
+
+namespace qi = boost::spirit::qi;
+namespace lex = boost::spirit::lex;
+
+enum tokenids
+{
+    IDWORD = lex::min_token_id, 
+    IDCHAR,
+    IDANY
+};
+
+template <typename Lexer>
+struct word_count_tokens : lex::lexer<Lexer>
+{
+    word_count_tokens()
+    {
+        this->self.add_pattern
+            ("TEST", "A")
+        ;
+
+        this->self = 
+                lex::string("{TEST}", IDWORD) 
+            |   lex::char_('a', IDCHAR)
+            |   lex::string(".", IDANY)
+            ;
+    }
+};
+
+template <typename Iterator>
+struct word_count_grammar : qi::grammar<Iterator>
+{
+    template <typename TokenDef>
+    word_count_grammar(TokenDef const& tok)
+      : word_count_grammar::base_type(start)
+      , w(0), c(0), a(0)
+    {
+        using boost::phoenix::ref;
+        using qi::token;
+
+        start =  *(   token(IDWORD) [++ref(w)]
+                  |   token(IDCHAR) [++ref(c)]
+                  |   token(IDANY)  [++ref(a)]
+                  )
+              ;
+    }
+    std::size_t w, c, a;
+    qi::rule<Iterator> start;
+};
+
+
+int main()
+{
+    typedef lex::lexertl::token<
+        const char*, boost::mpl::vector<std::string>
+    > token_type;
+
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+    typedef word_count_tokens<lexer_type>::iterator_type iterator_type;
+    word_count_tokens<lexer_type> word_count;          // Our lexer
+    word_count_grammar<iterator_type> g (word_count);  // Our parser
+
+    std::string str ("AaBCD");
+    char const* first = str.c_str();
+    char const* last = &first[str.size()];
+
+    BOOST_TEST(lex::tokenize_and_parse(first, last, word_count, g));
+    BOOST_TEST(g.w == 1 && g.c == 1 && g.a == 3);
+
+    return boost::report_errors();
+}