Boost logo

Boost-Commit :

Subject: [Boost-commit] svn:boost r54691 - in trunk/boost/spirit/home/lex: . lexer lexer/lexertl
From: hartmut.kaiser_at_[hidden]
Date: 2009-07-05 17:29:55


Author: hkaiser
Date: 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
New Revision: 54691
URL: http://svn.boost.org/trac/boost/changeset/54691

Log:
Spirit: More work on lexer placeholder _value
Text files modified:
   trunk/boost/spirit/home/lex/argument.hpp | 118 ++++++++++++++++++++++++++++++++++++---
   trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp | 21 ++++--
   trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp | 24 +++++--
   trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp | 6 ++
   trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp | 24 +++++--
   trunk/boost/spirit/home/lex/lexer/pass_flags.hpp | 3
   trunk/boost/spirit/home/lex/tokenize_and_parse.hpp | 14 +++-
   7 files changed, 172 insertions(+), 38 deletions(-)

Modified: trunk/boost/spirit/home/lex/argument.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/argument.hpp (original)
+++ trunk/boost/spirit/home/lex/argument.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -134,17 +134,16 @@
 
     ///////////////////////////////////////////////////////////////////////////
     // The value_getter is used to create the _value placeholder, which is a
- // Phoenix actor used to access or change the value of the current token.
+ // Phoenix actor used to access the value of the current token.
     //
- // This actor is invoked whenever the placeholder '_value' is used in a
- // lexer semantic action:
+ // This Phoenix actor is invoked whenever the placeholder '_value' is used
+ // as a rvalue inside a lexer semantic action:
     //
     // lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
- // this->self = identifier
- // [ _value = construct_<std::string>(_start, _end) ];
+ // this->self = identifier [ std::cout << _value ];
     //
- // The example shows how to use _value to set the identifier name as the
- // token value.
+ // The example shows how to use _value to print the identifier name (which
+ // is the initial token value).
     struct value_getter
     {
         typedef mpl::true_ no_nullary;
@@ -165,7 +164,73 @@
         typename result<Env>::type
         eval(Env const& env) const
         {
- return fusion::at_c<4>(env.args()).value();
+ return fusion::at_c<4>(env.args()).get_value();
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
+ // The value_setter is a Phoenix actor used to change the name of the
+ // current lexer state by calling set_state_name() on the context (which
+ // is the 4th parameter to any lexer semantic actions).
+ //
+ // This Phoenix actor is invoked whenever the placeholder '_value' is used
+ // as a lvalue inside a lexer semantic action:
+ //
+ // lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+ // this->self = identifier [ _value = "identifier" ];
+ //
+ // The example shows how to change the token value after matching a token
+ // 'identifier'.
+ template <typename Actor>
+ struct value_setter
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef void type;
+ };
+
+ template <typename Env>
+ void eval(Env const& env) const
+ {
+ fusion::at_c<4>(env.args()).set_value(actor_());
+ }
+
+ value_setter(Actor const& actor)
+ : actor_(actor) {}
+
+ // see explanation for this constructor at the end of this file
+ value_setter(phoenix::actor<value_getter>, Actor const& actor)
+ : actor_(actor) {}
+
+ Actor actor_;
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
+ // The value_context is used as a noop Phoenix actor to create the
+ // placeholder '_value' (see below). It is a noop actor because it is used
+ // as a placeholder only, while it is being converted either to a
+ // value_getter (if used as a rvalue) or to a value_setter (if used as a
+ // lvalue). The conversion is achieved by specializing and overloading a
+ // couple of the Phoenix templates from the Phoenix expression composition
+ // engine (see the end of this file).
+ struct value_context
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef unused_type type;
+ };
+
+ template <typename Env>
+ unused_type
+ eval(Env const& env) const
+ {
+ return unused;
         }
     };
 
@@ -223,7 +288,7 @@
 
     // '_value' may be used to access and change the token value of the current
     // token
- phoenix::actor<value_getter> const _value = value_getter();
+ phoenix::actor<value_context> const _value = value_context();
 
     // _state may be used to access and change the name of the current lexer
     // state
@@ -273,6 +338,41 @@
         typedef spirit::state_setter<typename as_actor<RHS>::type> type;
     };
 
+ ///////////////////////////////////////////////////////////////////////////
+ // The specialization of as_actor_base<> below is needed to convert all
+ // occurrences of _value in places where it's used as a rvalue into the
+ // proper Phoenix actor (spirit::value_getter) accessing the token value.
+ template<>
+ struct as_actor_base<actor<spirit::value_context> >
+ {
+ typedef spirit::value_getter type;
+
+ static spirit::value_getter
+ convert(actor<spirit::value_context>)
+ {
+ return spirit::value_getter();
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
+ // The specialization of as_composite<> below is needed to convert all
+ // assignments to _value (places where it's used as a lvalue) into the
+ // proper Phoenix actor (spirit::value_setter) allowing to change the
+ // token value.
+ template <typename RHS>
+ struct as_composite<assign_eval, actor<spirit::value_context>, RHS>
+ {
+ // For an assignment to _value (a spirit::value_context actor), this
+ // specialization makes Phoenix's compose() function construct a
+ // spirit::value_setter actor from 1. the LHS, a spirit::value_getter
+ // actor (due to the specialization of as_actor_base<> above),
+ // and 2. the RHS actor.
+ // This is why spirit::value_setter needs a constructor which takes
+ // a dummy spirit::value_getter as its first argument in addition
+ // to its real, second argument (the RHS actor).
+ typedef spirit::value_setter<typename as_actor<RHS>::type> type;
+ };
+
 }}
 
 #undef SPIRIT_DECLARE_ARG

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -139,6 +139,7 @@
                     return result = mp.ftor.eof;
 #endif
 
+ data.reset_value();
                 Iterator end = data.get_first();
                 std::size_t unique_id = boost::lexer::npos;
                 std::size_t id = data.next(end, unique_id);
@@ -187,18 +188,18 @@
                 BOOST_SCOPED_ENUM(pass_flags) pass =
                     data.invoke_actions(state, id, unique_id, end);
 
- if (pass_flags::pass_normal == pass) {
- // return matched token, advancing 'data.first_' past the
+ if (data.has_value()) {
+ // return matched token using the token value as set before
+ // using data.set_value(), advancing 'data.first_' past the
                     // matched sequence
                     assign_on_exit<Iterator> on_exit(data.get_first(), end);
- return result = result_type(id, state, data.get_first(), end);
+ return result = result_type(id, state, data.get_value());
                 }
- else if (pass_flags::pass_use_value == pass) {
- // return matched token using the token value as set before
- // using data.set_value(), advancing 'data.first_' past the
+ else if (pass_flags::pass_normal == pass) {
+ // return matched token, advancing 'data.first_' past the
                     // matched sequence
                     assign_on_exit<Iterator> on_exit(data.get_first(), end);
- return result = result_type(id, state, data.value());
+ return result = result_type(id, state, data.get_first(), end);
                 }
                 else if (pass_flags::pass_fail == pass) {
                     // if the data.first_ got adjusted above, revert this adjustment
@@ -234,6 +235,12 @@
         }
 
         template <typename MultiPass>
+ static std::size_t get_state(MultiPass& mp)
+ {
+ return mp.shared()->ftor.get_state();
+ }
+
+ template <typename MultiPass>
         static std::size_t
         map_state(MultiPass const& mp, char_type const* statename)
         {

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -148,7 +148,12 @@
             Iterator const& get_first() const { return first_; }
             Iterator const& get_last() const { return last_; }
 
- unused_type value() const { return unused; }
+ iterator_range<Iterator> get_value() const
+ {
+ return iterator_range<Iterator>(first_, last_);
+ }
+ bool has_value() const { return false; }
+ void reset_value() {}
 
         protected:
             Iterator& first_;
@@ -252,7 +257,8 @@
             template <typename IterData>
             data (IterData const& data, Iterator& first, Iterator const& last)
               : base_type(data, first, last)
- , actions_(data.actions_), hold_(), has_hold_(false) {}
+ , actions_(data.actions_), hold_(), has_hold_(false)
+ , has_value_(false) {}
 
             // invoke attached semantic actions, if defined
             BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
@@ -316,25 +322,27 @@
                 has_hold_ = true;
             }
 
- TokenValue const& value() const
- {
- return value_;
- }
- TokenValue& value()
+ TokenValue const& get_value() const
             {
+ if (!has_value_)
+ value_ = base_type::get_value();
                 return value_;
             }
             template <typename Value>
             void set_value(Value const& val)
             {
                 value_ = val;
+ has_value_ = true;
             }
+ bool has_value() const { return has_value_; }
+ void reset_value() { has_value_ = false; }
 
         protected:
             semantic_actions_type const& actions_;
             Iterator hold_; // iterator needed to support lex::more()
+ mutable TokenValue value_; // token value to use
             bool has_hold_; // 'true' if hold_ is valid
- TokenValue value_;
+ bool has_value_; // 'true' if value_ is valid
         };
     }
 

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -95,6 +95,12 @@
             return unique_functor_type::set_state(*this, state);
         }
 
+ // get the curent state for the underlying lexer object
+ std::size_t get_state()
+ {
+ return unique_functor_type::get_state(*this);
+ }
+
         // map the given state name to a corresponding state id as understood
         // by the underlying lexer object
         std::size_t map_state(char_type const* statename)

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -166,7 +166,12 @@
             Iterator const& get_first() const { return first_; }
             Iterator const& get_last() const { return last_; }
 
- unused_type value() const { return unused; }
+ iterator_range<Iterator> get_value() const
+ {
+ return iterator_range<Iterator>(first_, last_);
+ }
+ bool has_value() const { return false; }
+ void reset_value() {}
 
         protected:
             Iterator& first_;
@@ -274,7 +279,8 @@
             static_data (IterData const& data, Iterator& first
                   , Iterator const& last)
               : base_type(data, first, last)
- , actions_(data.actions_), hold_(), has_hold_(false) {}
+ , actions_(data.actions_), hold_(), has_hold_(false)
+ , has_value_(false) {}
 
             // invoke attached semantic actions, if defined
             BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
@@ -338,25 +344,27 @@
                 has_hold_ = true;
             }
 
- TokenValue const& value() const
- {
- return value_;
- }
- TokenValue& value()
+ TokenValue const& get_value() const
             {
+ if (!has_value_)
+ value_ = base_type::get_value();
                 return value_;
             }
             template <typename Value>
             void set_value(Value const& val)
             {
                 value_ = val;
+ has_value_ = true;
             }
+ bool has_value() const { return has_value_; }
+ void reset_value() { has_value_ = false; }
 
         protected:
             semantic_actions_type const& actions_;
             Iterator hold_; // iterator needed to support lex::more()
+ mutable TokenValue value_; // token value to use
             bool has_hold_; // 'true' if hold_ is valid
- TokenValue value_;
+ bool has_value_; // 'true' if value_ is valid
         };
     }
 

Modified: trunk/boost/spirit/home/lex/lexer/pass_flags.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/pass_flags.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/pass_flags.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -19,8 +19,7 @@
     {
         pass_fail = 0, // make the current match fail in retrospective
         pass_normal = 1, // continue normal token matching, that's the default
- pass_ignore = 2, // ignore the current token and start matching the next
- pass_use_value = 3 // continue normal matching but use token value as set before
+ pass_ignore = 2 // ignore the current token and start matching the next
     };
     BOOST_SCOPED_ENUM_END
 

Modified: trunk/boost/spirit/home/lex/tokenize_and_parse.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/tokenize_and_parse.hpp (original)
+++ trunk/boost/spirit/home/lex/tokenize_and_parse.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -251,11 +251,10 @@
     ///////////////////////////////////////////////////////////////////////////
     namespace detail
     {
- template <typename Token>
- bool tokenize_callback(Token const& t, void (*f)(Token const&))
+ template <typename Token, typename F>
+ bool tokenize_callback(Token const& t, F f)
         {
- f(t);
- return true;
+ return f(t);
         }
 
         template <typename Token, typename Eval>
@@ -266,6 +265,13 @@
         }
 
         template <typename Token>
+ bool tokenize_callback(Token const& t, void (*f)(Token const&))
+ {
+ f(t);
+ return true;
+ }
+
+ template <typename Token>
         bool tokenize_callback(Token const& t, bool (*f)(Token const&))
         {
             return f(t);


Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk