Boost logo

Boost-Commit :

From: hartmut.kaiser_at_[hidden]
Date: 2008-04-17 21:12:11


Author: hkaiser
Date: 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
New Revision: 44528
URL: http://svn.boost.org/trac/boost/changeset/44528

Log:
Spirit.Lex: Added support for token ids other than std::size_t.
Text files modified:
   trunk/boost/spirit/home/lex/lexer/lexer.hpp | 24 ++++++++++++---------
   trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp | 3 +
   trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp | 4 +++
   trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp | 3 ++
   trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp | 9 +------
   trunk/boost/spirit/home/lex/lexer/string_token_def.hpp | 4 ++
   trunk/boost/spirit/home/lex/lexer/token_def.hpp | 44 +++++++++++++++++++++++++++------------
   trunk/boost/spirit/home/lex/lexer/token_set.hpp | 20 ++++++++++--------
   8 files changed, 69 insertions(+), 42 deletions(-)

Modified: trunk/boost/spirit/home/lex/lexer/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexer.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -47,6 +47,8 @@
             typedef typename proto::terminal<terminal_holder>::type tag;
             typedef proto::extends<tag, lexer_def_> base_type;
 
+ typedef typename LexerDef::id_type id_type;
+
             tag make_tag() const
             {
                 tag xpr = {{ this }};
@@ -65,7 +67,7 @@
                 // id, and the corresponding pair of iterators
                 typedef typename Iterator::base_iterator_type iterator_type;
                 typedef
- fusion::vector<std::size_t, iterator_range<iterator_type> >
+ fusion::vector<id_type, iterator_range<iterator_type> >
                 type;
             };
                 
@@ -84,7 +86,7 @@
                     token_type;
 
                     token_type &t = *first;
- if (0 != t.id()) {
+ if (token_is_valid(t)) {
                     // any of the token definitions matched
                         qi::detail::assign_to(t, attr);
                         ++first;
@@ -109,31 +111,31 @@
                 {}
 
                 adder const&
- operator()(char_type c, std::size_t token_id = 0) const
+ operator()(char_type c, id_type token_id = 0) const
                 {
                     if (0 == token_id)
- token_id = static_cast<std::size_t>(c);
+ token_id = static_cast<id_type>(c);
                     def.def.add_token (def.state.c_str(), lex::detail::escape(c),
                         token_id);
                     return *this;
                 }
                 adder const&
- operator()(string_type const& s, std::size_t token_id = 0) const
+ operator()(string_type const& s, id_type token_id = id_type()) const
                 {
                     if (0 == token_id)
- token_id = next_id();
+ token_id = next_id<id_type>::get();
                     def.def.add_token (def.state.c_str(), s, token_id);
                     return *this;
                 }
                 template <typename Attribute>
                 adder const&
- operator()(token_def<Attribute, char_type>& tokdef,
- std::size_t token_id = 0) const
+ operator()(token_def<Attribute, char_type, id_type>& tokdef,
+ id_type token_id = id_type()) const
                 {
                     // make sure we have a token id
                     if (0 == token_id) {
                         if (0 == tokdef.id()) {
- token_id = next_id();
+ token_id = next_id<id_type>::get();
                             tokdef.id(token_id);
                         }
                         else {
@@ -281,13 +283,14 @@
     class lexer_def : noncopyable, public Lexer
     {
     private:
- typedef lexer_def<Lexer> self_type;
+ typedef lexer_def self_type;
         
         // avoid warnings about using 'this' in constructor
         lexer_def& this_() { return *this; }
 
     public:
         typedef Lexer lexer_type;
+ typedef typename Lexer::id_type id_type;
         typedef detail::lexer_def_<self_type> token_set;
         typedef typename Lexer::char_type char_type;
         typedef std::basic_string<char_type> string_type;
@@ -317,6 +320,7 @@
         typedef typename Definition::lexer_type lexer_type;
         typedef typename Definition::char_type char_type;
         typedef typename Definition::iterator_type iterator_type;
+ typedef typename Definition::id_type id_type;
 
         lexer(Definition& token_def_)
           : token_def(token_def_)

Modified: trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -17,7 +17,8 @@
     ///////////////////////////////////////////////////////////////////////////
     // This component represents a token definition
     ///////////////////////////////////////////////////////////////////////////
- template<typename Attribute = unused_type, typename Char = char>
+ template<typename Attribute = unused_type, typename Char = char,
+ typename Idtype = std::size_t>
     class token_def;
 
     ///////////////////////////////////////////////////////////////////////////

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -83,6 +83,9 @@
         typedef std::basic_string<char_type> string_type;
 
     public:
+ typedef Token token_type;
+ typedef typename Token::id_type id_type;
+
         // interface for token definition management
         void add_token (char_type const* state, string_type const& tokendef,
             std::size_t token_id)
@@ -192,6 +195,7 @@
         // Every lexer type to be used as a lexer for Spirit has to conform to
         // a public interface .
         typedef Token token_type;
+ typedef typename Token::id_type id_type;
         typedef TokenSet token_set;
         typedef lexertl_iterator<Functor> iterator_type;
         

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -53,6 +53,8 @@
     // iterator_type The type of the iterator used to access the
     // underlying character stream.
     //
+ // id_type The type of the token id used.
+ //
     // methods
     // default constructor
     // This should initialize the token as an end of
@@ -107,6 +109,7 @@
     {
         typedef Iterator iterator_type;
         typedef mpl::false_ has_state;
+ typedef std::size_t id_type;
         
         // default constructed tokens correspond to EOI tokens
         lexertl_token()

Modified: trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -12,6 +12,7 @@
 #endif
 
 #include <boost/spirit/home/lex/domain.hpp>
+#include <boost/spirit/home/lex/lexer/lexer_fwd.hpp>
 #include <boost/spirit/home/lex/lexer/terminal_holder.hpp>
 #include <boost/spirit/home/support/placeholders.hpp>
 #include <boost/spirit/home/support/meta_grammar.hpp>
@@ -29,12 +30,6 @@
     struct string_token_def;
     struct char_token_def;
     
- template<typename Attribute, typename Char>
- class token_def;
-
- template <typename TokenSet>
- class token_set;
-
     struct lexer_meta_grammar;
 
     template <typename Expr, typename Enable>
@@ -52,7 +47,7 @@
             // token_def<>
             meta_grammar::terminal_rule<
                 lex::domain,
- terminal_holder<proto::_, lex::token_def<proto::_, proto::_> >,
+ terminal_holder<proto::_, lex::token_def<proto::_, proto::_, proto::_> >,
                 terminal_director
>,
             // token_set

Modified: trunk/boost/spirit/home/lex/lexer/string_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/string_token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/string_token_def.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -27,7 +27,9 @@
         collect(Component const& component, LexerDef& lexdef,
             String const& state)
         {
- lexdef.add_token (state.c_str(), subject(component), next_id());
+ typedef typename LexerDef::id_type id_type;
+ lexdef.add_token (state.c_str(), subject(component),
+ next_id<id_type>::get());
         }
     };
                 

Modified: trunk/boost/spirit/home/lex/lexer/token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/token_def.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -34,22 +34,37 @@
     };
     
     ///////////////////////////////////////////////////////////////////////////
- inline std::size_t next_id()
+ // The next_id template needs to be specialized for any non-default token
+ // id type used by a custom token type. It need to expose a function
+ // 'static Idtype get()' returning the next available token id each time
+ // it is called.
+ template <typename Idtype>
+ struct next_id;
+
+ ///////////////////////////////////////////////////////////////////////////
+ // Default specialization for the next_id template returning the next
+ // available token id.
+ template <>
+ struct next_id<std::size_t>
     {
- static std::size_t next_token_id = min_token_id;
- return next_token_id++;
- }
-
+ static std::size_t get()
+ {
+ static std::size_t next_token_id = min_token_id;
+ return next_token_id++;
+ }
+ };
+
     ///////////////////////////////////////////////////////////////////////////
     // This component represents a token definition
     ///////////////////////////////////////////////////////////////////////////
- template<typename Attribute, typename Char>
+ template<typename Attribute, typename Char, typename Idtype>
     class token_def
       : public proto::extends<
             typename make_terminal_holder<
- token_def<Attribute, Char>*, token_def<Attribute, Char>
+ token_def<Attribute, Char, Idtype>*,
+ token_def<Attribute, Char, Idtype>
>::type,
- token_def<Attribute, Char>
+ token_def<Attribute, Char, Idtype>
>
     {
     private:
@@ -132,23 +147,24 @@
         {
             token_state = lexdef.add_state(state.c_str());
             if (0 == token_id)
- token_id = next_id();
+ token_id = next_id<Idtype>::get();
             lexdef.add_token (state.c_str(), def, token_id);
         }
             
     public:
         typedef Char char_type;
+ typedef Idtype id_type;
         typedef std::basic_string<char_type> string_type;
         
         // Lex interface: constructing token definitions
         token_def()
           : base_type(make_tag()), token_id(0), token_state(~0)
         {}
- explicit token_def(char_type def_, std::size_t id_ = 0)
+ explicit token_def(char_type def_, Idtype id_ = Idtype())
           : base_type(make_tag()), def(lex::detail::escape(def_)),
             token_id(0 == id_ ? def_ : id_), token_state(~0)
         {}
- explicit token_def(string_type def_, std::size_t id_ = 0)
+ explicit token_def(string_type def_, Idtype id_ = Idtype())
           : base_type(make_tag()), def(def_), token_id(id_), token_state(~0)
         {}
         
@@ -167,14 +183,14 @@
         }
         
         // general accessors
- std::size_t id() const { return token_id; }
- void id(std::size_t id) { token_id = id; }
+ Idtype id() const { return token_id; }
+ void id(Idtype id) { token_id = id; }
         string_type const& definition() const { return def; }
         std::size_t state() const { return token_state; }
         
     private:
         string_type def;
- std::size_t token_id;
+ Idtype token_id;
         std::size_t token_state;
     };
 

Modified: trunk/boost/spirit/home/lex/lexer/token_set.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_set.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/token_set.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -42,7 +42,7 @@
         // avoid warnings about using 'this' in constructor
         token_set& this_() { return *this; }
 
- typedef token_set<TokenSet> self_type;
+ typedef token_set self_type;
         typedef TokenSet base_token_set;
 
         // initialize proto base class
@@ -57,6 +57,8 @@
         }
 
     public:
+ typedef typename TokenSet::id_type id_type;
+
         // Qi interface: metafunction calculating parser return type
         template <typename Component, typename Context, typename Iterator>
         struct attribute
@@ -65,7 +67,7 @@
             // and the corresponding pair of iterators
             typedef typename Iterator::base_iterator_type iterator_type;
             typedef
- fusion::vector<std::size_t, iterator_range<iterator_type> >
+ fusion::vector<id_type, iterator_range<iterator_type> >
             type;
         };
 
@@ -88,7 +90,7 @@
                 BOOST_ASSERT(~0 != token_state);
 
                 token_type &t = *first;
- if (0 != t.id() && token_state == t.state()) {
+ if (token_is_valid(t) && token_state == t.state()) {
                 // any of the token definitions matched
                     qi::detail::assign_to(t, attr);
                     ++first;
@@ -123,7 +125,7 @@
             {}
 
             adder const&
- operator()(char_type c, std::size_t token_id = 0) const
+ operator()(char_type c, id_type token_id = id_type()) const
             {
                 if (0 == token_id)
                     token_id = static_cast<std::size_t>(c);
@@ -132,22 +134,22 @@
                 return *this;
             }
             adder const&
- operator()(string_type const& s, std::size_t token_id = 0) const
+ operator()(string_type const& s, id_type token_id = id_type()) const
             {
                 if (0 == token_id)
- token_id = next_id();
+ token_id = next_id<id_type>::get();
                 def.add_token (def.initial_state().c_str(), s, token_id);
                 return *this;
             }
             template <typename Attribute>
             adder const&
- operator()(token_def<Attribute, char_type>& tokdef,
- std::size_t token_id = 0) const
+ operator()(token_def<Attribute, char_type, id_type>& tokdef,
+ id_type token_id = id_type()) const
             {
                 // make sure we have a token id
                 if (0 == token_id) {
                     if (0 == tokdef.id()) {
- token_id = next_id();
+ token_id = next_id<id_type>::get();
                         tokdef.id(token_id);
                     }
                     else {


Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk