[Boost-commit] svn:boost r57566 - in trunk/boost/spirit: home/lex/lexer/lexertl home/support/detail/lexer include

Subject: [Boost-commit] svn:boost r57566 - in trunk/boost/spirit: home/lex/lexer/lexertl home/support/detail/lexer include
From: hartmut.kaiser_at_[hidden]
Date: 2009-11-11 09:57:04

Author: hkaiser
Date: 2009-11-11 09:57:04 EST (Wed, 11 Nov 2009)
New Revision: 57566
URL: http://svn.boost.org/trac/boost/changeset/57566

Log:
Spirit: updating static lexer support
Text files modified: 
   trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp  |    62 +++++++++++++++++++++++++-------------- 
   trunk/boost/spirit/home/support/detail/lexer/generate_re2c.hpp |    52 +++++++++++++++++++++------------       
   trunk/boost/spirit/include/lex_generate_static_lexertl.hpp     |     6 +++                                     
   3 files changed, 79 insertions(+), 41 deletions(-)

Modified: trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp	(original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp	2009-11-11 09:57:04 EST (Wed, 11 Nov 2009)
@@ -17,6 +17,10 @@
 #include <boost/spirit/home/support/detail/lexer/size_t.hpp>
 #include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
 #include <boost/spirit/home/lex/lexer/lexertl/static_version.hpp>
+#include <boost/spirit/home/karma/numeric/uint.hpp>
+#include <boost/spirit/home/karma/string/lit.hpp>
+#include <boost/spirit/home/karma/operator/sequence.hpp>
+#include <boost/spirit/home/karma/generate.hpp>
 #include <boost/algorithm/string.hpp>
 #include <boost/lexical_cast.hpp>
 
@@ -567,21 +571,24 @@
             }
             else 
             {
-                result = "\\x";
-                char buffer[3];
-                result += ltoa(ch, buffer, 16);
+                typedef karma::uint_generator<unsigned char, 16> uintgen_type;
+                std::back_insert_iterator<std::string> sink(result);
+                karma::generate(sink, "\\x" << uintgen_type()(ch));
             }
             break;
         }
         return result;
     }
 
-    inline std::basic_string<wchar_t> get_charcode(wchar_t ch)
+    inline std::string get_charcode(wchar_t ch)
     {
-        // not implemented yet
-        std::basic_string<wchar_t> result;
-        result = ch;
-        return result;
+        if (ch & ~0xff) {
+            BOOST_ASSERT(false);    // not implemented yet
+
+            std::string result;
+            return result;
+        }
+        return  get_charcode(static_cast<char>(ch & 0xff));
     }
 
     ///////////////////////////////////////////////////////////////////////////
@@ -683,20 +690,24 @@
                     }
                 }
 
-                os_ << "    if (curr_ == end_) goto end;\n";
-                os_ << "    ch_ = *curr_;\n";
-                if (iter_->bol_index != boost::lexer::npos)
-                {
-                    os_ << "\n    if (bol) goto state" << dfa_ << '_' 
-                        << iter_->bol_index << ";\n";
-                }
-                if (iter_->eol_index != boost::lexer::npos)
-                {
-                    os_ << "\n    if (ch_ == '\n') goto state" << dfa_ 
-                        << '_' << iter_->eol_index << ";\n";
+                if (t_ < transitions_ || 
+                    iter_->bol_index != boost::lexer::npos ||
+                    iter_->eol_index != boost::lexer::npos)
+                {
+                    os_ << "    if (curr_ == end_) goto end;\n";
+                    os_ << "    ch_ = *curr_;\n";
+                    if (iter_->bol_index != boost::lexer::npos)
+                    {
+                        os_ << "\n    if (bol) goto state" << dfa_ << '_' 
+                            << iter_->bol_index << ";\n";
+                    }
+                    if (iter_->eol_index != boost::lexer::npos)
+                    {
+                        os_ << "\n    if (ch_ == '\n') goto state" << dfa_ 
+                            << '_' << iter_->eol_index << ";\n";
+                    }
+                    os_ << "    ++curr_;\n";
                 }
-                os_ << "    ++curr_;\n";
-
 
                 for (/**/; t_ < transitions_; ++t_)
                 {
@@ -725,7 +736,14 @@
                         {
                             if (!first_char_)
                             {
-                                os_ << " || ";
+                                if (iter_->token._negated)
+                                {
+                                    os_ << " && ";
+                                }
+                                else
+                                {
+                                    os_ << " || ";
+                                }
                             }
 
                             first_char_ = false;

Modified: trunk/boost/spirit/home/support/detail/lexer/generate_re2c.hpp
==============================================================================
--- trunk/boost/spirit/home/support/detail/lexer/generate_re2c.hpp	(original)
+++ trunk/boost/spirit/home/support/detail/lexer/generate_re2c.hpp	2009-11-11 09:57:04 EST (Wed, 11 Nov 2009)
@@ -63,12 +63,14 @@
     return result;
 }
 
-inline std::basic_string<wchar_t> get_charcode(wchar_t ch)
+inline std::string get_charcode(wchar_t ch)
 {
-    // not implemented yet
-    std::basic_string<wchar_t> result;
-    result = ch;
-    return result;
+    if (ch & ~0xff) 
+    {
+        std::string result;
+        return result;    // not implemented yet
+    }
+    return get_charcode(static_cast<char>(ch & 0xff));
 }
 
 template<typename CharT>
@@ -274,22 +276,27 @@
                 }
             }
 
-            os_ << "    if (curr_ == end_) goto end;\n";
-            os_ << "    ch_ = *curr_;\n";
-
-            if (iter_->bol_index != boost::lexer::npos)
+            if (t_ < transitions_ || 
+                iter_->bol_index != boost::lexer::npos ||
+                iter_->eol_index != boost::lexer::npos)
             {
-                os_ << "\n    if (bol_) goto state" << dfa_ << '_' <<
-                    iter_->bol_index << ";\n\n";
-            }
+                os_ << "    if (curr_ == end_) goto end;\n";
+                os_ << "    ch_ = *curr_;\n";
 
-            if (iter_->eol_index != boost::lexer::npos)
-            {
-                os_ << "\n    if (ch_ == '\n') goto state" << dfa_ << '_' <<
-                    iter_->eol_index << ";\n\n";
-            }
+                if (iter_->bol_index != boost::lexer::npos)
+                {
+                    os_ << "\n    if (bol_) goto state" << dfa_ << '_' <<
+                        iter_->bol_index << ";\n\n";
+                }
 
-            os_ << "    ++curr_;\n";
+                if (iter_->eol_index != boost::lexer::npos)
+                {
+                    os_ << "\n    if (ch_ == '\n') goto state" << dfa_ << '_' <<
+                        iter_->eol_index << ";\n\n";
+                }
+
+                os_ << "    ++curr_;\n";
+            }
 
             for (; t_ < transitions_; ++t_)
             {
@@ -319,7 +326,14 @@
                     {
                         if (!first_char_)
                         {
-                            os_ << " || ";
+                            if (iter_->token._negated)
+                            {
+                                os_ << " && ";
+                            }
+                            else
+                            {
+                                os_ << " || ";
+                            }
                         }
 
                         first_char_ = false;

Modified: trunk/boost/spirit/include/lex_generate_static_lexertl.hpp
==============================================================================
--- trunk/boost/spirit/include/lex_generate_static_lexertl.hpp	(original)
+++ trunk/boost/spirit/include/lex_generate_static_lexertl.hpp	2009-11-11 09:57:04 EST (Wed, 11 Nov 2009)
@@ -8,5 +8,11 @@
 =============================================================================*/
 #ifndef BOOST_SPIRIT_INCLUDE_LEX_LEXER_GENERATE_STATIC_LEXERTL
 #define BOOST_SPIRIT_INCLUDE_LEX_LEXER_GENERATE_STATIC_LEXERTL
+
+#if defined(_MSC_VER)
+#pragma once
+#endif
+
 #include <boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>
+
 #endif