[Boost-commit] svn:boost r51949 - in trunk/tools/quickbook: . test

Subject: [Boost-commit] svn:boost r51949 - in trunk/tools/quickbook: . test
From: daniel_james_at_[hidden]
Date: 2009-03-24 04:34:11

Author: danieljames
Date: 2009-03-24 04:34:09 EDT (Tue, 24 Mar 2009)
New Revision: 51949
URL: http://svn.boost.org/trac/boost/changeset/51949

Log:
Use our own escape character parser.

We're currently using the spirit escaped character parser. This is a
strict C parser and causes parse errors for code that we probably should
accept, such as any hexadecimal encoded characters with a value greater
than the maximum value of the current platform's 'char'.  This is \x7F
on some platforms so it rejects '\x80' upwards. As well as rejecting
characters that might be valid, it also means that quickbook will act
differently on different platforms.

In python strings, '\xaaa' is equivalent to '\xaa' + 'a', but the spirit
parser interprets this as a character with value '0xaaa'. So we probably
should accept these.

I also think we should be liberal about what we accept. IMO it isn't our
job to enforce correct C++/python, just to create a reasonable
rendering of our input. So rather than write a parser which understands
the various types of escapes, I just wrote one that ignores any
character following a backslash.

Refs #2860.
Added:
   trunk/tools/quickbook/test/code-block-3.gold   (contents, props changed)
   trunk/tools/quickbook/test/code-block-3.quickbook   (contents, props changed)
Text files modified: 
   trunk/tools/quickbook/syntax_highlight.hpp |    22 +++++++++++++---------                  
   trunk/tools/quickbook/test/Jamfile.v2      |     1 +                                       
   2 files changed, 14 insertions(+), 9 deletions(-)

Modified: trunk/tools/quickbook/syntax_highlight.hpp
==============================================================================
--- trunk/tools/quickbook/syntax_highlight.hpp	(original)
+++ trunk/tools/quickbook/syntax_highlight.hpp	2009-03-24 04:34:09 EDT (Tue, 24 Mar 2009)
@@ -14,7 +14,6 @@
 #include <boost/spirit/include/classic_confix.hpp>
 #include <boost/spirit/include/classic_chset.hpp>
 #include <boost/spirit/include/classic_symbols.hpp>
-#include <boost/spirit/include/classic_escape_char.hpp>
 #include <boost/spirit/include/classic_loops.hpp>
 #include "./phrase.hpp"
 
@@ -127,12 +126,14 @@
                     =   +chset_p("~!%^&*()+={[}]:;,<.>?/|\\-")
                     ;
 
+                string_char = ('\\' >> anychar_p) | (anychar_p - '\\');
+
                 string_
-                    =   !as_lower_d['l'] >> confix_p('"', *c_escape_ch_p, '"')
+                    =   !as_lower_d['l'] >> confix_p('"', *string_char, '"')
                     ;
 
                 char_
-                    =   !as_lower_d['l'] >> confix_p('\'', *c_escape_ch_p, '\'')
+                    =   !as_lower_d['l'] >> confix_p('\'', *string_char, '\'')
                     ;
 
                 number
@@ -150,7 +151,8 @@
             }
 
             rule<Scanner>   program, macro, preprocessor, comment, special, string_, 
-                            char_, number, identifier, keyword, qbk_phrase, escape;
+                            char_, number, identifier, keyword, qbk_phrase, escape,
+                            string_char;
 
             symbols<> keyword_;
             phrase_grammar<EscapeActions> common;
@@ -275,16 +277,18 @@
                     =   ! string_prefix >> (long_string | short_string)
                     ;
 
+                string_char = ('\\' >> anychar_p) | (anychar_p - '\\');
+            
                 short_string
-                    =   confix_p('\'', * c_escape_ch_p, '\'') |
-                        confix_p('"', * c_escape_ch_p, '"')    
+                    =   confix_p('\'', * string_char, '\'') |
+                        confix_p('"', * string_char, '"')
                     ;
             
                 long_string
                     // Note: the "str_p" on the next two lines work around
                     // an INTERNAL COMPILER ERROR when using VC7.1
-                    =   confix_p(str_p("'''"), * lex_escape_ch_p, "'''") |
-                        confix_p(str_p("\"\"\""), * lex_escape_ch_p, "\"\"\"")
+                    =   confix_p(str_p("'''"), * string_char, "'''") |
+                        confix_p(str_p("\"\"\""), * string_char, "\"\"\"")
                     ;
                 
                 number
@@ -303,7 +307,7 @@
 
             rule<Scanner>   program, macro, comment, special, string_, string_prefix, 
                             short_string, long_string, number, identifier, keyword, 
-                            qbk_phrase, escape;
+                            qbk_phrase, escape, string_char;
 
             symbols<> keyword_;
             phrase_grammar<EscapeActions> common;

Modified: trunk/tools/quickbook/test/Jamfile.v2
==============================================================================
--- trunk/tools/quickbook/test/Jamfile.v2	(original)
+++ trunk/tools/quickbook/test/Jamfile.v2	2009-03-24 04:34:09 EDT (Tue, 24 Mar 2009)
@@ -14,6 +14,7 @@
     [ quickbook-test quickbook-manual ]
     [ quickbook-test code-block-1 ]
     [ quickbook-test code-block-2 ]
+    [ quickbook-test code-block-3 ]
     [ quickbook-test code-snippet ]
     [ quickbook-test preformatted ]
     [ quickbook-test link-side-by-side ]

Added: trunk/tools/quickbook/test/code-block-3.gold
==============================================================================
--- (empty file)
+++ trunk/tools/quickbook/test/code-block-3.gold	2009-03-24 04:34:09 EDT (Tue, 24 Mar 2009)
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE library PUBLIC "-//Boost//DTD BoostBook XML V1.0//EN" "http://www.boost.org/tools/boostbook/dtd/boostbook.dtd">
+<article id="code_block_3" last-revision="DEBUG MODE Date: 2000/12/20 12:00:00 $"
+ xmlns:xi="http://www.w3.org/2001/XInclude">
+  <title>Code Block 3</title>
+  <articleinfo>
+  </articleinfo>
+  <section id="code_block_3.python_code_block">
+    <title>Python code block</title>
+    <para>
+      
+<programlisting><phrase role="keyword">print</phrase> <phrase role="string">&quot;\xfabln\xeck&quot;</phrase>
+</programlisting>
+    </para>
+  </section>
+  <section id="code_block_3.c___code_block">
+    <title>C++ code block</title>
+    <para>
+      This isn't valid C++ but I think we should accept it;
+    </para>
+    <para>
+      [cpp] 
+<programlisting><phrase role="identifier">std</phrase><phrase role="special">::</phrase><phrase role="identifier">cout</phrase><phrase role="special">&lt;&lt;</phrase><phrase role="string">&quot;\xfabln\xeck&quot;</phrase><phrase role="special">&lt;&lt;</phrase><phrase role="string">&quot;\n&quot;</phrase><phrase role="special">;</phrase>
+</programlisting>
+    </para>
+  </section>
+</article>

Added: trunk/tools/quickbook/test/code-block-3.quickbook
==============================================================================
--- (empty file)
+++ trunk/tools/quickbook/test/code-block-3.quickbook	2009-03-24 04:34:09 EDT (Tue, 24 Mar 2009)
@@ -0,0 +1,21 @@
+[article Code Block 3]
+
+[section Python code block]
+
+[python]
+``
+    print "\xfabln\xeck"
+``
+
+[endsect]
+
+[section C++ code block]
+
+This isn't valid C++ but I think we should accept it;
+
+[cpp]
+``
+    std::cout<<"\xfabln\xeck"<<"\n";
+``
+
+[endsect]