[Boost-commit] svn:boost r76790 - branches/quickbook-dev/tools/quickbook/src

Subject: [Boost-commit] svn:boost r76790 - branches/quickbook-dev/tools/quickbook/src
From: dnljms_at_[hidden]
Date: 2012-01-29 19:06:37

Author: danieljames
Date: 2012-01-29 19:06:36 EST (Sun, 29 Jan 2012)
New Revision: 76790
URL: http://svn.boost.org/trac/boost/changeset/76790

Log:
Quickbook: Fix handling UTF-8 characters in the syntax highlighter.

So far, quickbook has just pretended that it's dealing with a single
byte encoding.  For the most part this works quite well due to the
design of UTF-8, but the syntax highlighter needs to pick out individual
characters correctly, so implement just enough to do that. Does odd
things with invalid UTF-8.
Text files modified: 
   branches/quickbook-dev/tools/quickbook/src/parsers.hpp          |    42 ++++++++++++++++++++++++++++++++++++++++
   branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp |    10 ++++----                                
   2 files changed, 47 insertions(+), 5 deletions(-)

Modified: branches/quickbook-dev/tools/quickbook/src/parsers.hpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/parsers.hpp	(original)
+++ branches/quickbook-dev/tools/quickbook/src/parsers.hpp	2012-01-29 19:06:36 EST (Sun, 29 Jan 2012)
@@ -253,6 +253,48 @@
     };
     
     lookback_gen const lookback = lookback_gen();
+ 
+    ///////////////////////////////////////////////////////////////////////////
+    //
+    // UTF-8 code point
+    //
+    // Very crude, it doesn't check that the code point is in any way valid.
+    // Just looks for the beginning of the next character. This is just for
+    // implementing some crude fixes, rather than full unicode support. I'm
+    // sure experts would be appalled.
+    //
+    ///////////////////////////////////////////////////////////////////////////
+
+    struct utf8_char_parser : public cl::parser<utf8_char_parser>
+    {
+        typedef utf8_char_parser self_t;
+
+        template <typename Scanner>
+        struct result
+        {
+            typedef cl::match<> type;
+        };
+
+        template <typename Scanner>
+        typename result<Scanner>::type parse(Scanner const& scan) const
+        {
+            typedef typename Scanner::iterator_t iterator_t;
+
+            if (scan.at_end()) return scan.no_match();
+
+            iterator_t save(scan.first);
+
+            do {
+                ++scan.first;
+            } while (!scan.at_end() &&
+                    ((unsigned char) *scan.first & 0xc0) == 0x80);
+
+            return scan.create_match(scan.first.base() - save.base(),
+                    cl::nil_t(), save, scan.first);
+        }
+    };
+  
+    utf8_char_parser const utf8_char_p = utf8_char_parser();
 }
 
 #endif // BOOST_QUICKBOOK_SCOPED_BLOCK_HPP

Modified: branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp	(original)
+++ branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp	2012-01-29 19:06:36 EST (Sun, 29 Jan 2012)
@@ -287,7 +287,7 @@
                     |   string_                         [span("string")]
                     |   char_                           [span("char")]
                     |   number                          [span("number")]
-                    |   cl::repeat_p(1)[cl::anychar_p]  [unexpected_char]
+                    |   utf8_char_p                     [unexpected_char]
                     )
                     ;
 
@@ -362,7 +362,7 @@
                     =   +cl::chset_p("~!%^&*()+={[}]:;,<.>?/|\\-")
                     ;
 
-                string_char = ('\\' >> cl::anychar_p) | (cl::anychar_p - '\\');
+                string_char = ('\\' >> utf8_char_p) | (cl::anychar_p - '\\');
 
                 string_
                     =   !cl::as_lower_d['l'] >> cl::confix_p('"', *string_char, '"')
@@ -442,7 +442,7 @@
                     |   special                         [span("special")]
                     |   string_                         [span("string")]
                     |   number                          [span("number")]
-                    |   cl::repeat_p(1)[cl::anychar_p]  [unexpected_char]
+                    |   utf8_char_p                     [unexpected_char]
                     )
                     ;
 
@@ -498,7 +498,7 @@
                     =   ! string_prefix >> (long_string | short_string)
                     ;
 
-                string_char = ('\\' >> cl::anychar_p) | (cl::anychar_p - '\\');
+                string_char = ('\\' >> utf8_char_p) | (cl::anychar_p - '\\');
             
                 short_string
                     =   cl::confix_p('\'', * string_char, '\'') |
@@ -564,7 +564,7 @@
                     =
                     *(  macro
                     |   escape          
-                    |   cl::repeat_p(1)[cl::anychar_p]  [plain_char]
+                    |   utf8_char_p                     [plain_char]
                     )
                     ;