[Boost-commit] svn:boost r60740 - in trunk/libs/spirit/example/scheme: . detail

Subject: [Boost-commit] svn:boost r60740 - in trunk/libs/spirit/example/scheme: . detail
From: joel_at_[hidden]
Date: 2010-03-20 22:59:41

Author: djowel
Date: 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
New Revision: 60740
URL: http://svn.boost.org/trac/boost/changeset/60740

Log:
Straight UTF-8 implementation
Text files modified: 
   trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp |     3 ++                                      
   trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp |    16 +++++++++++++-                          
   trunk/libs/spirit/example/scheme/out.txt                  |     2                                         
   trunk/libs/spirit/example/scheme/sexpr.hpp                |    42 ++++++++++++++++++--------------------- 
   trunk/libs/spirit/example/scheme/sexpr_test.cpp           |     2                                         
   trunk/libs/spirit/example/scheme/utree.hpp                |     8 +++++++                                 
   6 files changed, 46 insertions(+), 27 deletions(-)

Modified: trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp	(original)
+++ trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp	2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -115,6 +115,9 @@
 
         utree_type::info get_type() const;
         void set_type(utree_type::info t);
+        int get_subtype() const;
+        void set_subtype(int t);
+
         std::size_t size() const;
         char const* str() const;
 

Modified: trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp	(original)
+++ trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp	2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -11,12 +11,24 @@
 {
     inline utree_type::info fast_string::get_type() const
     {
-        return static_cast<utree_type::info>(buff[small_string_size]);
+        return static_cast<utree_type::info>(buff[small_string_size] & 0x7);
     }
 
     inline void fast_string::set_type(utree_type::info t)
     {
-        buff[small_string_size] = static_cast<char>(t);
+        buff[small_string_size] = static_cast<char>(t)
+                                | (buff[small_string_size] & 0xF8);
+    }
+
+    inline int fast_string::get_subtype() const
+    {
+        return buff[small_string_size] >> 3;
+    }
+
+    inline void fast_string::set_subtype(int t)
+    {
+        buff[small_string_size] = (t << 3)
+                                | (buff[small_string_size] & 0x7);
     }
 
     inline std::size_t fast_string::size() const

Modified: trunk/libs/spirit/example/scheme/out.txt
==============================================================================
--- trunk/libs/spirit/example/scheme/out.txt	(original)
+++ trunk/libs/spirit/example/scheme/out.txt	2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -1,2 +1,2 @@
-success: (123.45 true false 255 63 "this is a € string" "Τη γλώσσα μου έδωσαν ελληνική" b0123456789abcdef0123456789abcdef (92 ("another string" apple Sîne)))
+success: (123.45 true false 255 63 "this is a € string" "Τη γλώσσα μου έδωσαν ελληνική" b0123456789abcdef123456789abcdef (92 ("another string" apple Sîne)))
 

Modified: trunk/libs/spirit/example/scheme/sexpr.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/sexpr.hpp	(original)
+++ trunk/libs/spirit/example/scheme/sexpr.hpp	2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -9,8 +9,6 @@
 
 #include <string>
 
-#define BOOST_SPIRIT_UNICODE // We'll use unicode (UTF8) all throughout
-
 #include <boost/config/warning_disable.hpp>
 #include <boost/cstdint.hpp>
 #include <boost/spirit/include/qi.hpp>
@@ -24,9 +22,8 @@
 
 namespace scheme
 {
-    using boost::spirit::unicode::char_;
-    using boost::spirit::unicode::space;
-    using boost::spirit::unicode::print;
+    using boost::spirit::ascii::char_;
+    using boost::spirit::ascii::space;
     using boost::spirit::qi::grammar;
     using boost::spirit::qi::rule;
     using boost::spirit::qi::eol;
@@ -46,11 +43,10 @@
     using boost::spirit::qi::lit;
     using boost::phoenix::function;
 
-    typedef boost::spirit::char_encoding::unicode unicode;
     typedef boost::uint32_t uchar; // a unicode code point
 
     template <typename Iterator>
-    struct white_space : grammar<Iterator, unicode>
+    struct white_space : grammar<Iterator>
     {
         white_space() : white_space::base_type(start)
         {
@@ -60,7 +56,7 @@
                 ;
         }
 
-        rule<Iterator, unicode> start;
+        rule<Iterator> start;
     };
 
     namespace detail
@@ -79,17 +75,17 @@
             }
         };
 
-        struct push_symbol_utf8
+        struct push_symbol
         {
             template <typename S, typename C>
             struct result { typedef void type; };
 
-            void operator()(std::string& utf8, uchar code_point) const
+            void operator()(std::string& utf8, char ch) const
             {
                 if (utf8.size() == 0)
                     utf8 += '\0';   //  mark a symbol with prefix 0
                                     //  (a 0 byte at the beginning signifies a symbol)
-                push_utf8()(utf8, code_point);
+                utf8 += ch;
             }
         };
 
@@ -129,7 +125,7 @@
     }
 
     template <typename Iterator>
-    struct string : grammar<Iterator, unicode, std::string()>
+    struct string : grammar<Iterator, std::string()>
     {
         string() : string::base_type(start)
         {
@@ -148,23 +144,23 @@
 
             start
                 = '"'
-                >> *(str_esc(_val) | (char_ - '"')  [push_utf8(_val, _1)])
+                >> *(str_esc(_val) | (~char_('"'))  [_val += _1])
                 >> '"'
                 ;
         }
 
-        rule<Iterator, unicode, void(std::string&)> str_esc;
-        rule<Iterator, unicode, std::string()> start;
+        rule<Iterator, void(std::string&)> str_esc;
+        rule<Iterator, std::string()> start;
     };
 
     template <typename Iterator>
-    struct sexpr : grammar<Iterator, unicode, white_space<Iterator>, utree()>
+    struct sexpr : grammar<Iterator, white_space<Iterator>, utree()>
     {
         sexpr() : sexpr::base_type(start)
         {
             real_parser<double, strict_real_policies<double> > strict_double;
             uint_parser<unsigned char, 16, 2, 2> hex2;
-            function<detail::push_symbol_utf8> push_symbol_utf8;
+            function<detail::push_symbol> push_symbol;
             function<detail::push_binary> push_binary;
 
             start   = atom | list;
@@ -178,8 +174,8 @@
                     | symbol                            [_val = _1]
                     ;
 
-            char const* exclude = " ();\"\n\r\t";
-            symbol  = +lexeme[print - char_(exclude)]   [push_symbol_utf8(_val, _1)];
+            char const* exclude = " ();\"\0-\31\127";
+            symbol  = +lexeme[~char_(exclude)]          [push_symbol(_val, _1)];
 
             number  = strict_double                     [_val = _1]
                     | lexeme[no_case["0x"] >> hex]      [_val = _1]
@@ -190,10 +186,10 @@
             byte_str = lexeme[no_case['b'] >> +(hex2    [push_binary(_val, _1)])];
         }
 
-        rule<Iterator, unicode, white_space<Iterator>, utree()> start, list;
-        rule<Iterator, unicode, utree()> atom, number;
-        rule<Iterator, unicode, std::string()> symbol;
-        rule<Iterator, unicode, std::string()> byte_str;
+        rule<Iterator, white_space<Iterator>, utree()> start, list;
+        rule<Iterator, utree()> atom, number;
+        rule<Iterator, std::string()> symbol;
+        rule<Iterator, std::string()> byte_str;
         scheme::string<Iterator> string;
     };
 }

Modified: trunk/libs/spirit/example/scheme/sexpr_test.cpp
==============================================================================
--- trunk/libs/spirit/example/scheme/sexpr_test.cpp	(original)
+++ trunk/libs/spirit/example/scheme/sexpr_test.cpp	2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -58,7 +58,7 @@
         std::istream_iterator<char>(),
         std::back_inserter(source_code));
 
-    typedef boost::u8_to_u32_iterator<std::string::const_iterator> iterator_type;
+    typedef std::string::const_iterator iterator_type;
     iterator_type first(source_code.begin());
     iterator_type last(source_code.end());
 

Modified: trunk/libs/spirit/example/scheme/utree.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/utree.hpp	(original)
+++ trunk/libs/spirit/example/scheme/utree.hpp	2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -50,6 +50,14 @@
 
         struct nil {};
 
+        struct binary
+        {
+            binary(unsigned char bits[], std::size_t len)
+              : bits(bits), len(len) {}
+            unsigned char* bits;
+            std::size_t len;
+        };
+
         utree();
         explicit utree(bool b);
         explicit utree(unsigned int i);