[Boost-commit] svn:boost r72202 - in branches/release: . boost boost/spirit boost/spirit/home boost/spirit/home/lex/lexer/lexertl

Subject: [Boost-commit] svn:boost r72202 - in branches/release: . boost boost/spirit boost/spirit/home boost/spirit/home/lex/lexer/lexertl
From: hartmut.kaiser_at_[hidden]
Date: 2011-05-26 22:04:29

Author: hkaiser
Date: 2011-05-26 22:04:28 EDT (Thu, 26 May 2011)
New Revision: 72202
URL: http://svn.boost.org/trac/boost/changeset/72202

Log:
Spirit: merging from trunk
Properties modified: 
   branches/release/   (props changed)
   branches/release/boost/   (props changed)
   branches/release/boost/spirit/   (props changed)
   branches/release/boost/spirit/home/   (props changed)
Text files modified: 
   branches/release/boost/spirit/home/lex/lexer/lexertl/functor.hpp             |     3                                         
   branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp        |     2                                         
   branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp |   163 +++++++++++++++++++++++++++++++++++++-- 
   3 files changed, 158 insertions(+), 10 deletions(-)

Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/functor.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/functor.hpp	(original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/functor.hpp	2011-05-26 22:04:28 EDT (Thu, 26 May 2011)
@@ -196,6 +196,9 @@
                 // data.first_ back to the start of the previously matched token.
                 bool adjusted = data.adjust_start();
 
+                // set the end of the matched input sequence in the token data
+                data.set_end(end);
+
                 // invoke attached semantic actions, if defined, might change
                 // state, id, data.first_, and/or end
                 BOOST_SCOPED_ENUM(pass_flags) pass = 

Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp	(original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp	2011-05-26 22:04:28 EDT (Thu, 26 May 2011)
@@ -168,6 +168,8 @@
             std::size_t get_state() const { return 0; }
             void set_state(std::size_t) {}
 
+            void set_end(Iterator const& it) {}
+
             Iterator& get_first() { return first_; }
             Iterator const& get_first() const { return first_; }
             Iterator const& get_last() const { return last_; }

Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp	(original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp	2011-05-26 22:04:28 EDT (Thu, 26 May 2011)
@@ -112,7 +112,7 @@
             //
             // This function does nothing as long as no semantic actions are 
             // used.
-            Iterator const& less(Iterator const& it, int n) 
+            Iterator const& less(Iterator const& it, int) 
             { 
                 // The following assertion fires most likely because you are 
                 // using lexer semantic actions without using the actor_lexer
@@ -150,7 +150,7 @@
             //
             // This function does nothing as long as no semantic actions are 
             // used.
-            bool lookahead(std::size_t id, std::size_t state = std::size_t(~0)) 
+            bool lookahead(std::size_t, std::size_t /*state*/ = std::size_t(~0)) 
             { 
                 // The following assertion fires most likely because you are 
                 // using lexer semantic actions without using the actor_lexer
@@ -184,6 +184,8 @@
             std::size_t get_state() const { return 0; }
             void set_state(std::size_t) {}
 
+            void set_end(Iterator const& it) {}
+
             Iterator& get_first() { return first_; }
             Iterator const& get_first() const { return first_; }
             Iterator const& get_last() const { return last_; }
@@ -204,7 +206,7 @@
             next_token_functor next_token_;
             get_state_name_type get_state_name_;
 
-            bool bol_;
+            bool bol_;      // helper storing whether last character was \n
 
         private:
             // silence MSVC warning C4512: assignment operator could not be generated
@@ -212,7 +214,7 @@
         };
 
         ///////////////////////////////////////////////////////////////////////
-        //  doesn't support no actors, but does support states
+        //  doesn't support lexer semantic actions, but supports state
         template <typename Iterator, typename TokenValue>
         class static_data<Iterator, mpl::false_, mpl::true_, TokenValue>
           : public static_data<Iterator, mpl::false_, mpl::false_, TokenValue>
@@ -297,7 +299,8 @@
                 semantic_actions_type;
 
         protected:
-            typedef static_data<Iterator, mpl::false_, HasState, TokenValue> base_type;
+            typedef static_data<Iterator, mpl::false_, HasState, TokenValue> 
+                base_type;
             typedef typename base_type::char_type char_type;
             typedef typename semantic_actions_type::functor_wrapper_type
                 functor_wrapper_type;
@@ -338,7 +341,7 @@
             // token back to the input stream). 
             Iterator const& less(Iterator& it, int n) 
             {
-                it = this->first_;
+                it = this->get_first();
                 std::advance(it, n);
                 return it;
             }
@@ -350,7 +353,7 @@
             // the current token value rather than replacing it.
             void more()
             {
-                hold_ = this->first_;
+                hold_ = this->get_first();
                 has_hold_ = true;
             }
 
@@ -368,7 +371,7 @@
                     state = this->state_;
 
                 return id == this->next_token_(
-                    state, bol, end, this->last_, unique_id);
+                    state, bol, end, this->get_eoi(), unique_id);
             }
 
             // The adjust_start() and revert_adjust_start() are helper 
@@ -379,14 +382,14 @@
                 if (!has_hold_)
                     return false;
 
-                std::swap(this->first_, hold_);
+                std::swap(this->get_first(), hold_);
                 has_hold_ = false;
                 return true;
             }
             void revert_adjust_start()
             {
                 // this will be called only if adjust_start above returned true
-                std::swap(this->first_, hold_);
+                std::swap(this->get_first(), hold_);
                 has_hold_ = true;
             }
 
@@ -423,6 +426,146 @@
             // silence MSVC warning C4512: assignment operator could not be generated
             static_data& operator= (static_data const&);
         };
+
+        ///////////////////////////////////////////////////////////////////////
+        //  does support lexer semantic actions, may support state, is used for
+        //  position_token exposing exactly one type
+        template <typename Iterator, typename HasState, typename TokenValue>
+        class static_data<Iterator, mpl::true_, HasState, boost::optional<TokenValue> > 
+          : public static_data<Iterator, mpl::false_, HasState, TokenValue>
+        {
+        public:
+            typedef semantic_actions<Iterator, HasState, static_data> 
+                semantic_actions_type;
+
+        protected:
+            typedef static_data<Iterator, mpl::false_, HasState, TokenValue> 
+                base_type;
+            typedef typename base_type::char_type char_type;
+            typedef typename semantic_actions_type::functor_wrapper_type
+                functor_wrapper_type;
+
+        public:
+            typedef Iterator base_iterator_type;
+            typedef boost::optional<TokenValue> token_value_type;
+            typedef boost::optional<TokenValue> const& get_value_type;
+            typedef typename base_type::state_type state_type;
+            typedef typename base_type::state_name_type state_name_type;
+
+            typedef detail::wrap_action<functor_wrapper_type
+              , Iterator, static_data, std::size_t> wrap_action_type;
+
+            template <typename IterData>
+            static_data (IterData const& data_, Iterator& first, Iterator const& last)
+              : base_type(data_, first, last)
+              , actions_(data_.actions_), hold_()
+              , has_value_(false), has_hold_(false) 
+            {
+                spirit::traits::assign_to(first, last, value_);
+                has_value_ = true;
+            }
+
+            // invoke attached semantic actions, if defined
+            BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
+              , std::size_t& id, std::size_t unique_id, Iterator& end)
+            {
+                return actions_.invoke_actions(state, id, unique_id, end, *this); 
+            }
+
+            // The function less() is used by the implementation of the support
+            // function lex::less(). Its functionality is equivalent to flex'
+            // function yyless(): it returns an iterator positioned to the 
+            // nth input character beyond the current start iterator (i.e. by
+            // assigning the return value to the placeholder '_end' it is 
+            // possible to return all but the first n characters of the current 
+            // token back to the input stream). 
+            Iterator const& less(Iterator& it, int n) 
+            {
+                it = this->get_first();
+                std::advance(it, n);
+                return it;
+            }
+
+            // The function more() is used by the implementation of the support 
+            // function lex::more(). Its functionality is equivalent to flex'
+            // function yymore(): it tells the lexer that the next time it 
+            // matches a rule, the corresponding token should be appended onto 
+            // the current token value rather than replacing it.
+            void more()
+            {
+                hold_ = this->get_first();
+                has_hold_ = true;
+            }
+
+            // The function lookahead() is used by the implementation of the 
+            // support function lex::lookahead. It can be used to implement 
+            // lookahead for lexer engines not supporting constructs like flex'
+            // a/b  (match a, but only when followed by b)
+            bool lookahead(std::size_t id, std::size_t state = std::size_t(~0))
+            {
+                Iterator end = end_;
+                std::size_t unique_id = boost::lexer::npos;
+                bool bol = this->bol_;
+
+                if (std::size_t(~0) == state)
+                    state = this->state_;
+
+                return id == this->next_token_(
+                    state, bol, end, this->get_eoi(), unique_id);
+            }
+
+            // The adjust_start() and revert_adjust_start() are helper 
+            // functions needed to implement the functionality required for 
+            // lex::more(). It is called from the functor body below.
+            bool adjust_start()
+            {
+                if (!has_hold_)
+                    return false;
+
+                std::swap(this->get_first(), hold_);
+                has_hold_ = false;
+                return true;
+            }
+            void revert_adjust_start()
+            {
+                // this will be called only if adjust_start above returned true
+                std::swap(this->get_first(), hold_);
+                has_hold_ = true;
+            }
+
+            TokenValue const& get_value() const 
+            {
+                if (!has_value_) {
+                    spirit::traits::assign_to(this->get_first(), end_, value_);
+                    has_value_ = true;
+                }
+                return value_;
+            }
+            template <typename Value>
+            void set_value(Value const& val)
+            {
+                value_ = val;
+                has_value_ = true;
+            }
+            void set_end(Iterator const& it)
+            {
+                end_ = it;
+            }
+            bool has_value() const { return has_value_; }
+            void reset_value() { has_value_ = false; }
+
+        protected:
+            semantic_actions_type const& actions_;
+            Iterator hold_;     // iterator needed to support lex::more()
+            Iterator end_;      // iterator pointing to end of matched token
+            mutable token_value_type value_;  // token value to use
+            mutable bool has_value_;    // 'true' if value_ is valid
+            bool has_hold_;     // 'true' if hold_ is valid
+
+        private:
+            // silence MSVC warning C4512: assignment operator could not be generated
+            static_data& operator= (static_data const&);
+        };
     }
 }}}}