|
|
Boost-Commit : |
Subject: [Boost-commit] svn:boost r56142 - in trunk/boost/spirit/home/lex: . lexer lexer/lexertl
From: hartmut.kaiser_at_[hidden]
Date: 2009-09-10 14:38:20
Author: hkaiser
Date: 2009-09-10 14:38:19 EDT (Thu, 10 Sep 2009)
New Revision: 56142
URL: http://svn.boost.org/trac/boost/changeset/56142
Log:
Spirit: fixing lexer issue inhibiting usage of more than one instance of a static lexer
Text files modified:
trunk/boost/spirit/home/lex/lexer/action.hpp | 2 +-
trunk/boost/spirit/home/lex/lexer/lexer.hpp | 24 ++++++++++++++++++++----
trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp | 2 +-
trunk/boost/spirit/home/lex/lexer/string_token_def.hpp | 2 +-
trunk/boost/spirit/home/lex/lexer/token_def.hpp | 33 +--------------------------------
trunk/boost/spirit/home/lex/lexer_type.hpp | 2 +-
6 files changed, 25 insertions(+), 40 deletions(-)
Modified: trunk/boost/spirit/home/lex/lexer/action.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/action.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/action.hpp 2009-09-10 14:38:19 EDT (Thu, 10 Sep 2009)
@@ -31,7 +31,7 @@
template <typename LexerDef, typename String>
void collect(LexerDef& lexdef, String const& state) const
{
- // first collect the token definition information for the token_def
+ // collect the token definition information for the token_def
// this action is attached to
subject.collect(lexdef, state);
}
Modified: trunk/boost/spirit/home/lex/lexer/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexer.hpp 2009-09-10 14:38:19 EDT (Thu, 10 Sep 2009)
@@ -135,7 +135,7 @@
, id_type token_id = id_type()) const
{
if (id_type() == token_id)
- token_id = next_id<id_type>::get();
+ token_id = def.def.get_next_id();
def.def.add_token (def.state.c_str(), s, token_id);
return *this;
}
@@ -148,7 +148,7 @@
// make sure we have a token id
if (id_type() == token_id) {
if (id_type() == tokdef.id()) {
- token_id = next_id<id_type>::get();
+ token_id = def.def.get_next_id();
tokdef.id(token_id);
}
else {
@@ -280,7 +280,8 @@
// modes of the lexer
struct match_flags
{
- enum enum_type {
+ enum enum_type
+ {
match_default = 0, // no flags
match_not_dot_newline = 1, // the regex '.' doesn't match newlines
match_icase = 2 // all matching operations are case insensitive
@@ -290,6 +291,13 @@
///////////////////////////////////////////////////////////////////////////
// This represents a lexer object
///////////////////////////////////////////////////////////////////////////
+
+ // this is the first token id automatically assigned by the library if needed
+ enum tokenids
+ {
+ min_token_id = 0x10000
+ };
+
template <typename Lexer>
class lexer : public Lexer
{
@@ -297,6 +305,8 @@
// avoid warnings about using 'this' in constructor
lexer& this_() { return *this; }
+ typename Lexer::id_type next_token_id;
+
public:
typedef Lexer lexer_type;
typedef typename Lexer::id_type id_type;
@@ -308,7 +318,10 @@
typedef std::basic_string<char_type> string_type;
lexer(unsigned int flags = match_flags::match_default)
- : lexer_type(flags), self(this_(), lexer_type::initial_state()) {}
+ : lexer_type(flags)
+ , next_token_id(min_token_id)
+ , self(this_(), lexer_type::initial_state())
+ {}
// access iterator interface
template <typename Iterator>
@@ -321,6 +334,9 @@
std::size_t map_state(char_type const* state)
{ return this->lexer_type::add_state(state); }
+ // create a unique token id
+ id_type get_next_id() { return next_token_id++; }
+
lexer_def self; // allow for easy token definition
};
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp 2009-09-10 14:38:19 EDT (Thu, 10 Sep 2009)
@@ -93,7 +93,7 @@
suffix += name_suffix;
generate_delimiter(os_);
- os_ << "// this defines a generic accessor for the information above\n";
+ os_ << "// this defines a generic accessors for the information above\n";
os_ << "struct lexer" << suffix << "\n{\n";
os_ << " // return the number of lexer states\n";
os_ << " static std::size_t const state_count()\n";
Modified: trunk/boost/spirit/home/lex/lexer/string_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/string_token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/string_token_def.hpp 2009-09-10 14:38:19 EDT (Thu, 10 Sep 2009)
@@ -65,7 +65,7 @@
{
typedef typename LexerDef::id_type id_type;
if (std::size_t(~0) == id_)
- id_ = next_id<id_type>::get();
+ id_ = lexdef.get_next_id();
unique_id_ = lexdef.add_token (state.c_str(), str_, id_);
}
Modified: trunk/boost/spirit/home/lex/lexer/token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/token_def.hpp 2009-09-10 14:38:19 EDT (Thu, 10 Sep 2009)
@@ -33,37 +33,6 @@
namespace boost { namespace spirit { namespace lex
{
///////////////////////////////////////////////////////////////////////////
- // create a unique token id, note this is not thread safe
- ///////////////////////////////////////////////////////////////////////////
- enum tokenids
- {
- // this is the first token id automatically assigned by the library
- // if needed
- min_token_id = 0x10000
- };
-
- ///////////////////////////////////////////////////////////////////////////
- // The next_id template needs to be specialized for any non-default token
- // id type used by a custom token type. It needs to expose a function
- // 'static Idtype get()' returning the next available token id each time
- // it is called.
- template <typename Idtype>
- struct next_id;
-
- ///////////////////////////////////////////////////////////////////////////
- // Default specialization for the next_id template returning the next
- // available token id.
- template <>
- struct next_id<std::size_t>
- {
- static std::size_t get()
- {
- static std::size_t next_token_id = min_token_id;
- return next_token_id++;
- }
- };
-
- ///////////////////////////////////////////////////////////////////////////
// This component represents a token definition
///////////////////////////////////////////////////////////////////////////
template<typename Attribute = unused_type
@@ -160,7 +129,7 @@
token_state_ = state_id;
if (0 == token_id_)
- token_id_ = next_id<Idtype>::get();
+ token_id_ = lexdef.get_next_id();
if (0 == def_.which()) {
unique_id_ = lexdef.add_token(state.c_str()
Modified: trunk/boost/spirit/home/lex/lexer_type.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer_type.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer_type.hpp 2009-09-10 14:38:19 EDT (Thu, 10 Sep 2009)
@@ -26,7 +26,7 @@
// Requirement: l.collect(def, state) -> void
//
// l: a lexer
- // def: token definition
+ // def: token definition container
// state: lexer state this token definition needs to be added to
Derived const& derived() const
Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk