Boost logo

Boost-Commit :

Subject: [Boost-commit] svn:boost r65314 - in trunk: boost/wave boost/wave/grammars boost/wave/util libs/wave libs/wave/src libs/wave/test/testwave libs/wave/test/testwave/testfiles
From: hartmut.kaiser_at_[hidden]
Date: 2010-09-05 18:33:26


Author: hkaiser
Date: 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
New Revision: 65314
URL: http://svn.boost.org/trac/boost/changeset/65314

Log:
Wave: improved consistency in reporting skipped tokens to the hooks function when processing conditional preprocessing directives. Added a new test case verifying the skipped_token hook gets called reproducibly (t_2_020.cpp). Started to work on C++0x features for Wave.
Added:
   trunk/libs/wave/test/testwave/testfiles/t_2_020.cpp (contents, props changed)
Text files modified:
   trunk/boost/wave/grammars/cpp_defined_grammar.hpp | 7 ++-
   trunk/boost/wave/grammars/cpp_grammar.hpp | 36 ++++++++++++++---------
   trunk/boost/wave/grammars/cpp_predef_macros_grammar.hpp | 16 ++++++---
   trunk/boost/wave/token_ids.hpp | 60 +++++++++++++++++++++++++-------------
   trunk/boost/wave/util/cpp_iterator.hpp | 62 +++++++++++++++++++++++++++++----------
   trunk/boost/wave/util/cpp_macromap.hpp | 39 ++++++++++++++++--------
   trunk/boost/wave/util/insert_whitespace_detection.hpp | 5 +++
   trunk/boost/wave/util/interpret_pragma.hpp | 12 ++++---
   trunk/boost/wave/wave_version.hpp | 6 +-
   trunk/libs/wave/ChangeLog | 13 ++++++++
   trunk/libs/wave/src/token_ids.cpp | 2
   trunk/libs/wave/test/testwave/collect_hooks_information.hpp | 48 +++++++++++++++++++++++++++---
   trunk/libs/wave/test/testwave/testfiles/t_1_037.cpp | 13 ++-----
   trunk/libs/wave/test/testwave/testfiles/test.cfg | 1
   trunk/libs/wave/test/testwave/testwave_app.cpp | 8 +++++
   15 files changed, 231 insertions(+), 97 deletions(-)

Modified: trunk/boost/wave/grammars/cpp_defined_grammar.hpp
==============================================================================
--- trunk/boost/wave/grammars/cpp_defined_grammar.hpp (original)
+++ trunk/boost/wave/grammars/cpp_defined_grammar.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -85,15 +85,16 @@
                     [
                         spirit_append_actor(self.result_seq)
                     ]
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType, TokenTypeMask|PPTokenFlag)
                     [
                         spirit_append_actor(self.result_seq)
                     ]
- | pattern_p(OperatorTokenType|AltExtTokenType, ExtTokenTypeMask)
+ | pattern_p(OperatorTokenType|AltExtTokenType,
+ ExtTokenTypeMask|PPTokenFlag)
                     [
                         spirit_append_actor(self.result_seq)
                     ]
- | pattern_p(BoolLiteralTokenType, TokenTypeMask)
+ | pattern_p(BoolLiteralTokenType, TokenTypeMask|PPTokenFlag)
                     [
                         spirit_append_actor(self.result_seq)
                     ]

Modified: trunk/boost/wave/grammars/cpp_grammar.hpp
==============================================================================
--- trunk/boost/wave/grammars/cpp_grammar.hpp (original)
+++ trunk/boost/wave/grammars/cpp_grammar.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -331,10 +331,12 @@
>> +ppsp
                     ]
>> ( ch_p(T_IDENTIFIER)
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType,
+ TokenTypeMask|PPTokenFlag)
                         | pattern_p(OperatorTokenType|AltExtTokenType,
- ExtTokenTypeMask) // and, bit_and etc.
- | pattern_p(BoolLiteralTokenType, TokenTypeMask) // true/false
+ ExtTokenTypeMask|PPTokenFlag) // and, bit_and etc.
+ | pattern_p(BoolLiteralTokenType,
+ TokenTypeMask|PPTokenFlag) // true/false
                         )
>> ( ( no_node_d[eps_p(ch_p(T_LEFTPAREN))]
>> macro_parameters
@@ -353,10 +355,12 @@
                         no_node_d[ch_p(T_LEFTPAREN) >> *ppsp],
                        !list_p(
                             ( ch_p(T_IDENTIFIER)
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType,
+ TokenTypeMask|PPTokenFlag)
                             | pattern_p(OperatorTokenType|AltExtTokenType,
- ExtTokenTypeMask) // and, bit_and etc.
- | pattern_p(BoolLiteralTokenType, TokenTypeMask) // true/false
+ ExtTokenTypeMask|PPTokenFlag) // and, bit_and etc.
+ | pattern_p(BoolLiteralTokenType,
+ TokenTypeMask|PPTokenFlag) // true/false
 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
                             | ch_p(T_ELLIPSIS)
 #endif
@@ -384,10 +388,12 @@
>> +ppsp
                     ]
>> ( ch_p(T_IDENTIFIER)
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType,
+ TokenTypeMask|PPTokenFlag)
                         | pattern_p(OperatorTokenType|AltExtTokenType,
- ExtTokenTypeMask) // and, bit_and etc.
- | pattern_p(BoolLiteralTokenType, TokenTypeMask) // true/false
+ ExtTokenTypeMask|PPTokenFlag) // and, bit_and etc.
+ | pattern_p(BoolLiteralTokenType,
+ TokenTypeMask|PPTokenFlag) // true/false
                         )
                 ;
 
@@ -417,7 +423,7 @@
                     [
                         ch_p(T_PP_IF)
                         [ store_found_directive_type(self.found_directive) ]
- >> *ppsp
+// >> *ppsp
                     ]
>> +( anychar_p -
                             (ch_p(T_NEWLINE) | ch_p(T_CPPCOMMENT) | ch_p(T_EOF))
@@ -437,7 +443,7 @@
                     [
                         ch_p(T_PP_ELIF)
                         [ store_found_directive_type(self.found_directive) ]
- >> *ppsp
+// >> *ppsp
                     ]
>> +( anychar_p -
                             (ch_p(T_NEWLINE) | ch_p(T_CPPCOMMENT) | ch_p(T_EOF))
@@ -546,10 +552,12 @@
             ppqualifiedname
                 = no_node_d[*ppsp]
>> ( ch_p(T_IDENTIFIER)
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType,
+ TokenTypeMask|PPTokenFlag)
                         | pattern_p(OperatorTokenType|AltExtTokenType,
- ExtTokenTypeMask) // and, bit_and etc.
- | pattern_p(BoolLiteralTokenType, TokenTypeMask) // true/false
+ ExtTokenTypeMask|PPTokenFlag) // and, bit_and etc.
+ | pattern_p(BoolLiteralTokenType,
+ TokenTypeMask|PPTokenFlag) // true/false
                         )
                 ;
 

Modified: trunk/boost/wave/grammars/cpp_predef_macros_grammar.hpp
==============================================================================
--- trunk/boost/wave/grammars/cpp_predef_macros_grammar.hpp (original)
+++ trunk/boost/wave/grammars/cpp_predef_macros_grammar.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -76,10 +76,12 @@
         // make it possible to reuse the parse tree traversal code
             plain_define
                 = ( ch_p(T_IDENTIFIER)
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType,
+ TokenTypeMask|PPTokenFlag)
                     | pattern_p(OperatorTokenType|AltExtTokenType,
- ExtTokenTypeMask) // and, bit_and etc.
- | pattern_p(BoolLiteralTokenType, TokenTypeMask) // true/false
+ ExtTokenTypeMask|PPTokenFlag) // and, bit_and etc.
+ | pattern_p(BoolLiteralTokenType,
+ TokenTypeMask|PPTokenFlag) // true/false
                     )
>> !macro_parameters
>> !macro_definition
@@ -91,10 +93,12 @@
                         no_node_d[ch_p(T_LEFTPAREN) >> *ch_p(T_SPACE)],
                        !list_p(
                             ( ch_p(T_IDENTIFIER)
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType,
+ TokenTypeMask|PPTokenFlag)
                             | pattern_p(OperatorTokenType|AltExtTokenType,
- ExtTokenTypeMask) // and, bit_and etc.
- | pattern_p(BoolLiteralTokenType, TokenTypeMask) // true/false
+ ExtTokenTypeMask|PPTokenFlag) // and, bit_and etc.
+ | pattern_p(BoolLiteralTokenType,
+ TokenTypeMask|PPTokenFlag) // true/false
 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
                             | ch_p(T_ELLIPSIS)
 #endif

Modified: trunk/boost/wave/token_ids.hpp
==============================================================================
--- trunk/boost/wave/token_ids.hpp (original)
+++ trunk/boost/wave/token_ids.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -44,30 +44,31 @@
 ///////////////////////////////////////////////////////////////////////////////
 // the token_category helps to classify the different token types
 enum token_category {
- IdentifierTokenType = 0x10000000,
- ParameterTokenType = 0x11000000,
- ExtParameterTokenType = 0x11100000,
- KeywordTokenType = 0x20000000,
- OperatorTokenType = 0x30000000,
- LiteralTokenType = 0x40000000,
- IntegerLiteralTokenType = 0x41000000,
- FloatingLiteralTokenType = 0x42000000,
- StringLiteralTokenType = 0x43000000,
- CharacterLiteralTokenType = 0x44000000,
- BoolLiteralTokenType = 0x45000000,
- PPTokenType = 0x50000000,
- PPConditionalTokenType = 0x50800000,
+ IdentifierTokenType = 0x10080000,
+ ParameterTokenType = 0x11080000,
+ ExtParameterTokenType = 0x11180000,
+ KeywordTokenType = 0x20080000,
+ OperatorTokenType = 0x30080000,
+ LiteralTokenType = 0x40080000,
+ IntegerLiteralTokenType = 0x41080000,
+ FloatingLiteralTokenType = 0x42080000,
+ StringLiteralTokenType = 0x43080000,
+ CharacterLiteralTokenType = 0x44080000,
+ BoolLiteralTokenType = 0x45080000,
+ PPTokenType = 0x50080000,
+ PPConditionalTokenType = 0x50880000,
 
     UnknownTokenType = 0xA0000000,
     EOLTokenType = 0xB0000000,
     EOFTokenType = 0xC0000000,
     WhiteSpaceTokenType = 0xD0000000,
- InternalTokenType = 0xE0000000,
-
+ InternalTokenType = 0xE0080000,
+
     TokenTypeMask = 0xFF000000,
     AltTokenType = 0x00100000,
     TriGraphTokenType = 0x00200000,
     AltExtTokenType = 0x00500000, // and, bit_and etc.
+ PPTokenFlag = 0x00080000, // these are 'real' pp-tokens
     ExtTokenTypeMask = 0xFFF00000,
     ExtTokenOnlyMask = 0x00F00000,
     TokenValueMask = 0x000FFFFF,
@@ -262,7 +263,7 @@
     T_EOF = TOKEN_FROM_ID(401, EOFTokenType), // end of file reached
     T_EOI = TOKEN_FROM_ID(402, EOFTokenType), // end of input reached
     T_PP_NUMBER = TOKEN_FROM_ID(403, InternalTokenType),
-
+
 // MS extensions
     T_MSEXT_INT8 = TOKEN_FROM_ID(404, KeywordTokenType),
     T_MSEXT_INT16 = TOKEN_FROM_ID(405, KeywordTokenType),
@@ -287,7 +288,7 @@
     T_IMPORT = TOKEN_FROM_ID(421, KeywordTokenType),
 
     T_LAST_TOKEN_ID,
- T_LAST_TOKEN = ID_FROM_TOKEN(T_LAST_TOKEN_ID),
+ T_LAST_TOKEN = ID_FROM_TOKEN(T_LAST_TOKEN_ID & ~PPTokenFlag),
 
 // pseudo tokens to help streamlining macro replacement, these should not
 // returned from the lexer nor should these be returned from the pp-iterator
@@ -304,11 +305,15 @@
 #define TOKEN_FROM_ID(id, cat) boost::wave::token_id((id) | (cat))
 
 #undef ID_FROM_TOKEN
-#define ID_FROM_TOKEN(tok) ((tok) & ~boost::wave::TokenTypeMask)
+#define ID_FROM_TOKEN(tok) \
+ boost::wave::token_id((tok) & \
+ ~(boost::wave::TokenTypeMask|boost::wave::PPTokenFlag)) \
+ /**/
 
 #undef BASEID_FROM_TOKEN
 #define BASEID_FROM_TOKEN(tok) \
- boost::wave::token_id(((tok) & ~boost::wave::ExtTokenTypeMask)) \
+ boost::wave::token_id((tok) & \
+ ~(boost::wave::ExtTokenTypeMask|boost::wave::PPTokenFlag)) \
     /**/
 #define BASE_TOKEN(tok) \
     boost::wave::token_id((tok) & boost::wave::MainTokenMask) \
@@ -316,13 +321,26 @@
 #define CATEGORY_FROM_TOKEN(tok) ((tok) & boost::wave::TokenTypeMask)
 #define EXTCATEGORY_FROM_TOKEN(tok) ((tok) & boost::wave::ExtTokenTypeMask)
 #define IS_CATEGORY(tok, cat) \
- ((CATEGORY_FROM_TOKEN(tok) == (cat)) ? true : false) \
+ ((CATEGORY_FROM_TOKEN(tok) == CATEGORY_FROM_TOKEN(cat)) ? true : false) \
     /**/
 #define IS_EXTCATEGORY(tok, cat) \
- ((EXTCATEGORY_FROM_TOKEN(tok) == (cat)) ? true : false) \
+ ((EXTCATEGORY_FROM_TOKEN(tok) == EXTCATEGORY_FROM_TOKEN(cat)) ? true : false) \
     /**/
 
 ///////////////////////////////////////////////////////////////////////////////
+// verify whether the given token(-id) represents a valid pp_token
+inline bool is_pp_token(boost::wave::token_id id)
+{
+ return (id & boost::wave::PPTokenFlag) ? true : false;
+}
+
+template <typename TokenT>
+inline bool is_pp_token(TokenT const& tok)
+{
+ return is_pp_token(boost::wave::token_id(tok));
+}
+
+///////////////////////////////////////////////////////////////////////////////
 // return a token name
 BOOST_WAVE_DECL
 BOOST_WAVE_STRINGTYPE get_token_name(token_id tokid);

Modified: trunk/boost/wave/util/cpp_iterator.hpp
==============================================================================
--- trunk/boost/wave/util/cpp_iterator.hpp (original)
+++ trunk/boost/wave/util/cpp_iterator.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -306,9 +306,9 @@
     template <typename IteratorT>
     bool extract_identifier(IteratorT &it);
     template <typename IteratorT>
- bool ensure_is_last_on_line(IteratorT& it);
+ bool ensure_is_last_on_line(IteratorT& it, bool call_hook = true);
     template <typename IteratorT>
- bool skip_to_eol_with_check(IteratorT &it);
+ bool skip_to_eol_with_check(IteratorT &it, bool call_hook = true);
 
     bool pp_directive();
     template <typename IteratorT>
@@ -340,8 +340,7 @@
         typename parse_tree_type::const_iterator const &end);
     void on_elif(result_type const& found_directive,
         typename parse_tree_type::const_iterator const &begin,
- typename parse_tree_type::const_iterator const &end,
- token_sequence_type const& found_eoltokens);
+ typename parse_tree_type::const_iterator const &end);
     void on_error(typename parse_tree_type::const_iterator const &begin,
         typename parse_tree_type::const_iterator const &end);
 #if BOOST_WAVE_SUPPORT_WARNING_DIRECTIVE != 0
@@ -955,24 +954,42 @@
 
     ///////////////////////////////////////////////////////////////////////////
     template <typename ContextT, typename IteratorT>
- bool skip_to_eol(ContextT &ctx, IteratorT &it, IteratorT const &end)
+ bool skip_to_eol(ContextT &ctx, IteratorT &it, IteratorT const &end,
+ bool call_hook = true)
     {
         using namespace boost::wave;
 
         for (/**/; it != end; ++it) {
         token_id id = token_id(*it);
 
- call_skipped_token_hook(ctx, *it);
             if (T_CPPCOMMENT == id || T_NEWLINE == id ||
                 context_policies::util::ccomment_has_newline(*it))
             {
+ // always call hook for eol
+ call_skipped_token_hook(ctx, *it);
                 ++it; // skip eol/C/C++ comment
                 return true; // found eol
             }
+
+ if (call_hook)
+ call_skipped_token_hook(ctx, *it);
         }
         return false;
     }
 
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename ContextT, typename ContainerT>
+ inline void
+ remove_leading_whitespace(ContextT &ctx, ContainerT& c, bool call_hook = true)
+ {
+ typename ContainerT::iterator it = c.begin();
+ while (IS_CATEGORY(*it, WhiteSpaceTokenType)) {
+ typename ContainerT::iterator save = it++;
+ if (call_hook)
+ call_skipped_token_hook(ctx, *save);
+ c.erase(save);
+ }
+ }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -1006,9 +1023,9 @@
 template <typename ContextT>
 template <typename IteratorT>
 inline bool
-pp_iterator_functor<ContextT>::ensure_is_last_on_line(IteratorT& it)
+pp_iterator_functor<ContextT>::ensure_is_last_on_line(IteratorT& it, bool call_hook)
 {
- if (!impl::pp_is_last_on_line(ctx, it, iter_ctx->last, false))
+ if (!impl::pp_is_last_on_line(ctx, it, iter_ctx->last, call_hook))
     {
     // enable error recovery (start over with the next line)
         impl::skip_to_eol(ctx, it, iter_ctx->last);
@@ -1043,10 +1060,10 @@
 template <typename ContextT>
 template <typename IteratorT>
 inline bool
-pp_iterator_functor<ContextT>::skip_to_eol_with_check(IteratorT &it)
+pp_iterator_functor<ContextT>::skip_to_eol_with_check(IteratorT &it, bool call_hook)
 {
     typename ContextT::string_type value ((*it).get_value());
- if (!impl::skip_to_eol(ctx, it, iter_ctx->last) &&
+ if (!impl::skip_to_eol(ctx, it, iter_ctx->last, call_hook) &&
         !need_single_line(ctx.get_language()))
     {
     // The line doesn't end with an eol but eof token.
@@ -1075,8 +1092,8 @@
 {
     token_id id = token_id(*it);
     bool can_exit = true;
+ bool call_hook_in_skip = true;
     if (!ctx.get_if_block_status()) {
- impl::call_skipped_token_hook(ctx, *it);
         if (IS_EXTCATEGORY(*it, PPConditionalTokenType)) {
         // simulate the if block hierarchy
             switch (static_cast<unsigned int>(id)) {
@@ -1124,6 +1141,9 @@
                 break;
             }
         }
+ else {
+ impl::call_skipped_token_hook(ctx, *it);
+ }
     }
     else {
     // try to handle the simple pp directives without parsing
@@ -1209,6 +1229,7 @@
             {
                 on_undefine(it);
             }
+ call_hook_in_skip = false;
             break;
 
         case T_PP_IFDEF: // #ifdef
@@ -1217,6 +1238,7 @@
             {
                 on_ifdef(directive, it);
             }
+ call_hook_in_skip = false;
             break;
 
         case T_PP_IFNDEF: // #ifndef
@@ -1225,6 +1247,7 @@
             {
                 on_ifndef(directive, it);
             }
+ call_hook_in_skip = false;
             break;
 
 #if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0
@@ -1243,7 +1266,7 @@
 
 // start over with the next line, if only possible
     if (can_exit) {
- skip_to_eol_with_check(it);
+ skip_to_eol_with_check(it, call_hook_in_skip);
         return true; // may be safely ignored
     }
     return false; // do not ignore this pp directive
@@ -1432,7 +1455,7 @@
         break;
 
     case T_PP_ELIF: // #elif
- on_elif(found_directive, begin_child_it, end_child_it, found_eoltokens);
+ on_elif(found_directive, begin_child_it, end_child_it);
         break;
 
 // case T_PP_ELSE: // #else
@@ -1482,6 +1505,10 @@
         }
         break;
     }
+
+ // properly skip trailing newline for all directives
+ typename token_sequence_type::const_iterator eol = found_eoltokens.begin();
+ impl::skip_to_eol(ctx, eol, found_eoltokens.end());
     return true; // return newline only
 }
 
@@ -1948,6 +1975,8 @@
         make_ref_transform_iterator(end, get_value),
         std::inserter(toexpand, toexpand.end()));
 
+ impl::remove_leading_whitespace(ctx, toexpand);
+
 bool if_status = false;
 grammars::value_error status = grammars::error_noerror;
 token_sequence_type expanded;
@@ -2022,8 +2051,7 @@
 pp_iterator_functor<ContextT>::on_elif(
     result_type const& found_directive,
     typename parse_tree_type::const_iterator const &begin,
- typename parse_tree_type::const_iterator const &end,
- token_sequence_type const& found_eoltokens)
+ typename parse_tree_type::const_iterator const &end)
 {
 // preprocess the given sequence into the provided list
 get_token_value<result_type, parse_node_type> get_value;
@@ -2033,6 +2061,8 @@
         make_ref_transform_iterator(end, get_value),
         std::inserter(toexpand, toexpand.end()));
 
+ impl::remove_leading_whitespace(ctx, toexpand);
+
 // check current if block status
     if (ctx.get_if_block_some_part_status()) {
         if (!ctx.enter_elif_block(false)) {
@@ -2044,10 +2074,8 @@
 
     // skip all the expression and the trailing whitespace
     typename token_sequence_type::iterator begin2 = toexpand.begin();
- typename token_sequence_type::const_iterator begin3 = found_eoltokens.begin();
 
         impl::skip_to_eol(ctx, begin2, toexpand.end());
- impl::skip_to_eol(ctx, begin3, found_eoltokens.end());
         return; // one of previous #if/#elif was true, so don't enter this #elif
     }
 

Modified: trunk/boost/wave/util/cpp_macromap.hpp
==============================================================================
--- trunk/boost/wave/util/cpp_macromap.hpp (original)
+++ trunk/boost/wave/util/cpp_macromap.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -970,7 +970,7 @@
         typename ContainerT::size_type i;
 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
         bool is_ellipsis = false;
-
+
             if (IS_EXTCATEGORY((*cit), ExtParameterTokenType)) {
                 BOOST_ASSERT(boost::wave::need_variadics(ctx.get_language()));
                 i = token_id(*cit) - T_EXTPARAMETERBASE;
@@ -981,10 +981,10 @@
             {
                 i = token_id(*cit) - T_PARAMETERBASE;
             }
-
+
             BOOST_ASSERT(i < arguments.size());
             if (use_replaced_arg) {
-
+
 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
                 if (is_ellipsis) {
                 position_type const &pos = (*cit).get_position();
@@ -1636,7 +1636,13 @@
         lang);
     lexer_type end = lexer_type();
     for (/**/; it != end && T_EOF != token_id(*it); ++it)
+ {
+ // as of Wave V2.0.7 pasting of tokens is valid only if the resulting
+ // tokens are pp_tokens (as mandated by C++0x)
+ if (!is_pp_token(*it))
+ return false;
         rescanned.push_back(*it);
+ }
 
 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
     if (boost::wave::need_variadics(ctx.get_language()))
@@ -1644,8 +1650,6 @@
 #endif
 
 // test if the newly generated token sequence contains more than 1 token
-// the second one is the T_EOF token
-// BOOST_ASSERT(T_EOF == token_id(rescanned.back()));
     return 1 == rescanned.size();
 }
 
@@ -1655,6 +1659,22 @@
 // token sequence.
 //
 ///////////////////////////////////////////////////////////////////////////////
+template <typename Context>
+inline void report_invalid_concatenation(Context& ctx,
+ typename Context::token_type const& prev,
+ typename Context::token_type const& next,
+ typename Context::position_type const& main_pos)
+{
+typename Context::string_type error_string("\"");
+
+ error_string += prev.get_value();
+ error_string += "\" and \"";
+ error_string += next.get_value();
+ error_string += "\"";
+ BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, invalid_concat,
+ error_string.c_str(), main_pos);
+}
+
 template <typename ContextT>
 template <typename ContainerT>
 inline bool
@@ -1725,14 +1745,7 @@
                 !IS_CATEGORY(*prev, WhiteSpaceTokenType) &&
                 !IS_CATEGORY(*next, WhiteSpaceTokenType))
             {
- string_type error_string("\"");
-
- error_string += (*prev).get_value();
- error_string += "\" and \"";
- error_string += (*next).get_value();
- error_string += "\"";
- BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, invalid_concat,
- error_string.c_str(), main_pos);
+ report_invalid_concatenation(ctx, *prev, *next, main_pos);
                 return false;
             }
 

Modified: trunk/boost/wave/util/insert_whitespace_detection.hpp
==============================================================================
--- trunk/boost/wave/util/insert_whitespace_detection.hpp (original)
+++ trunk/boost/wave/util/insert_whitespace_detection.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -476,6 +476,11 @@
                 return true; // prevent ->*
             }
             break;
+
+ case T_POUND:
+ if (T_POUND == prev)
+ return true;
+ break;
         }
 
     // FIXME: else, handle operators separately (will catch to many cases)

Modified: trunk/boost/wave/util/interpret_pragma.hpp
==============================================================================
--- trunk/boost/wave/util/interpret_pragma.hpp (original)
+++ trunk/boost/wave/util/interpret_pragma.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -85,16 +85,18 @@
                                 [
                                     spirit_assign_actor(option)
                                 ]
- | pattern_p(KeywordTokenType, TokenTypeMask)
+ | pattern_p(KeywordTokenType,
+ TokenTypeMask|PPTokenFlag)
                                 [
                                     spirit_assign_actor(option)
                                 ]
                             | pattern_p(OperatorTokenType|AltExtTokenType,
- ExtTokenTypeMask) // and, bit_and etc.
+ ExtTokenTypeMask|PPTokenFlag) // and, bit_and etc.
                                 [
                                     spirit_assign_actor(option)
                                 ]
- | pattern_p(BoolLiteralTokenType, TokenTypeMask)
+ | pattern_p(BoolLiteralTokenType,
+ TokenTypeMask|PPTokenFlag)
                                 [
                                     spirit_assign_actor(option)
                                 ]
@@ -103,7 +105,7 @@
                                 ch_p(T_LEFTPAREN),
                                 ch_p(T_RIGHTPAREN)
                             )[spirit_assign_actor(values)],
- pattern_p(WhiteSpaceTokenType, TokenTypeMask)).hit)
+ pattern_p(WhiteSpaceTokenType, TokenTypeMask|PPTokenFlag)).hit)
             {
                 BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
                     ill_formed_pragma_option,
@@ -163,7 +165,7 @@
                                     *(anychar_p[spirit_append_actor(values)] - ch_p(T_NEWLINE))
                                 ]
                             ),
- pattern_p(WhiteSpaceTokenType, TokenTypeMask)
+ pattern_p(WhiteSpaceTokenType, TokenTypeMask|PPTokenFlag)
                        ).hit
                )
             {

Modified: trunk/boost/wave/wave_version.hpp
==============================================================================
--- trunk/boost/wave/wave_version.hpp (original)
+++ trunk/boost/wave/wave_version.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -16,11 +16,11 @@
 // BOOST_WAVE_VERSION & 0x0000FF is the sub-minor version
 // BOOST_WAVE_VERSION & 0x00FF00 is the minor version
 // BOOST_WAVE_VERSION & 0xFF0000 is the major version
-#define BOOST_WAVE_VERSION 0x020006
+#define BOOST_WAVE_VERSION 0x020100
 
 // The following defines contain the same information as above
 #define BOOST_WAVE_VERSION_MAJOR 2
-#define BOOST_WAVE_VERSION_MINOR 0
-#define BOOST_WAVE_VERSION_SUBMINOR 6
+#define BOOST_WAVE_VERSION_MINOR 1
+#define BOOST_WAVE_VERSION_SUBMINOR 0
 
 #endif // !defined(WAVE_VERSION_H_9D79ABDB_AC54_4C0A_89B1_F70A2DCFE21E_INCLUDED)

Modified: trunk/libs/wave/ChangeLog
==============================================================================
--- trunk/libs/wave/ChangeLog (original)
+++ trunk/libs/wave/ChangeLog 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -20,6 +20,19 @@
 
 CHANGELOG
 
+Boost V1.45.0
+- V2.1.0
+- Token pasting is well formed only as long as the formed token(s) are
+ pp_tokens as defined by the C++0x Standard. Until now, Wave allowed for
+ non-pp_tokens to be formed in --variadics mode.
+- Fixed a problem, which prevented reporting /##/ in a macro definition as
+ invalid token pasting.
+- Fixed problem preventing the skipped_token hook to be called for 'inactive'
+ conditional preprocessing directive tokens. Improved overall consistency in
+ reporting skipped tokens to the hooks function when processing conditional
+ preprocessing directives. Added a new test case verifying the skipped_token
+ hook gets called reproducibly (t_2_020.cpp).
+
 Boost V1.44.0
 - V2.0.6
 - Added information about the file type to iteration context. This can be

Modified: trunk/libs/wave/src/token_ids.cpp
==============================================================================
--- trunk/libs/wave/src/token_ids.cpp (original)
+++ trunk/libs/wave/src/token_ids.cpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -216,7 +216,7 @@
     BOOST_STATIC_ASSERT(
         sizeof(tok_names)/sizeof(tok_names[0]) == T_LAST_TOKEN-T_FIRST_TOKEN
     );
-
+
     unsigned int id = BASEID_FROM_TOKEN(tokid)-T_FIRST_TOKEN;
     return (id < T_LAST_TOKEN-T_FIRST_TOKEN) ? tok_names[id] : "<UnknownToken>";
 }

Modified: trunk/libs/wave/test/testwave/collect_hooks_information.hpp
==============================================================================
--- trunk/libs/wave/test/testwave/collect_hooks_information.hpp (original)
+++ trunk/libs/wave/test/testwave/collect_hooks_information.hpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -53,6 +53,35 @@
     return handle_filepath(pos.get_file()) + String("(") + linenum.c_str() + ")";
 }
 
+template <typename String>
+inline String repr(String const& value)
+{
+ String result;
+ typename String::const_iterator end = value.end();
+ for (typename String::const_iterator it = value.begin(); it != end; ++it)
+ {
+ typedef typename String::value_type char_type;
+ char_type c = *it;
+ if (c == static_cast<char_type>('\a'))
+ result.append("\\a");
+ else if (c == static_cast<char_type>('\b'))
+ result.append("\\b");
+ else if (c == static_cast<char_type>('\f'))
+ result.append("\\f");
+ else if (c == static_cast<char_type>('\n'))
+ result.append("\\n");
+ else if (c == static_cast<char_type>('\r'))
+ result.append("\\r");
+ else if (c == static_cast<char_type>('\t'))
+ result.append("\\t");
+ else if (c == static_cast<char_type>('\v'))
+ result.append("\\v");
+ else
+ result += static_cast<char_type>(c);
+ }
+ return result;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 template <typename Token>
 class collect_hooks_information
@@ -62,9 +91,14 @@
 
 public:
     collect_hooks_information(std::string& trace)
- : hooks_trace(trace)
+ : hooks_trace(trace), skipped_token_hooks(false)
     {}
 
+ void set_skipped_token_hooks(bool flag)
+ {
+ skipped_token_hooks = flag;
+ }
+
     ///////////////////////////////////////////////////////////////////////////
     //
     // The function 'expanding_function_like_macro' is called, whenever a
@@ -488,10 +522,13 @@
     void
     skipped_token(Context const& ctx, Token const& token)
     {
-// this generates a lot of noise
-// BOOST_WAVETEST_OSSTREAM strm;
-// strm << "12: " << std::endl;
-// hooks_trace += BOOST_WAVETEST_GETSTRING(strm);
+ // this normally generates a lot of noise
+ if (skipped_token_hooks) {
+ BOOST_WAVETEST_OSSTREAM strm;
+ strm << "12: " << repr(token.get_position()) << ": >"
+ << repr(token.get_value()) << "<" << std::endl;
+ hooks_trace += BOOST_WAVETEST_GETSTRING(strm);
+ }
     }
 
     ///////////////////////////////////////////////////////////////////////////
@@ -790,6 +827,7 @@
 
 private:
     std::string& hooks_trace;
+ bool skipped_token_hooks;
 };
 
 #endif

Modified: trunk/libs/wave/test/testwave/testfiles/t_1_037.cpp
==============================================================================
--- trunk/libs/wave/test/testwave/testfiles/t_1_037.cpp (original)
+++ trunk/libs/wave/test/testwave/testfiles/t_1_037.cpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -7,16 +7,11 @@
     LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 =============================================================================*/
 
-// the following concatenation failed
+// the following concatenation needs to fail (even if this construct is used
+// in some MS headers)
 
-
-//R #line 16 "t_1_037.cpp"
-//R // bool;
+//R
+//E t_1_037.cpp(16): error: pasting the following two tokens does not give a valid preprocessing token: "/" and "/"
 #define _VARIANT_BOOL /##/
 _VARIANT_BOOL bool;
 
-//H 10: t_1_037.cpp(15): #define
-//H 08: t_1_037.cpp(15): _VARIANT_BOOL=/##/
-//H 01: t_1_037.cpp(15): _VARIANT_BOOL
-//H 02: //
-//H 03: //

Added: trunk/libs/wave/test/testwave/testfiles/t_2_020.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/wave/test/testwave/testfiles/t_2_020.cpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -0,0 +1,261 @@
+/*=============================================================================
+ Boost.Wave: A Standard compliant C++ preprocessor library
+ http://www.boost.org/
+
+ Copyright (c) 2001-2010 Hartmut Kaiser. Distributed under the Boost
+ Software License, Version 1.0. (See accompanying file
+ LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+
+// check called hooks during conditional preprocessing
+//O --skipped_token_hooks
+
+#ifdef FOO
+int foo;
+#else
+float foo;
+#endif
+
+#if defined BAR
+int bar;
+#elif defined BAR2
+float bar;
+#endif
+
+#define BAZ
+#ifdef BAZ
+int baz;
+#else
+float baz;
+#endif
+
+#if defined BAZ
+int baz1;
+#elif defined BAZ
+float baz1;
+#endif
+
+#ifndef BAZ
+int baz2;
+#else
+float baz2;
+#endif
+
+#if 1
+int w1;
+#elif 1
+int w2;
+#else
+int w3;
+#endif
+
+#if 0
+int x1;
+#elif 1
+int x2;
+#else
+int x3;
+#endif
+
+#if 1
+int y1;
+#elif 0
+int y2;
+#else
+int y3;
+#endif
+
+#if 0
+int z1;
+#elif 0
+int z2;
+#else
+int z3;
+#endif
+
+//R #line 16 "t_2_020.cpp"
+//R float foo;
+//R #line 27 "t_2_020.cpp"
+//R int baz;
+//R #line 33 "t_2_020.cpp"
+//R int baz1;
+//R #line 41 "t_2_020.cpp"
+//R float baz2;
+//R #line 45 "t_2_020.cpp"
+//R int w1;
+//R #line 55 "t_2_020.cpp"
+//R int x2;
+//R #line 61 "t_2_020.cpp"
+//R int y1;
+//R #line 73 "t_2_020.cpp"
+//R int z3;
+
+//H 10: t_2_020.cpp(13): #ifdef
+//H 11: t_2_020.cpp(13): #ifdef FOO: 0
+//H 12: t_2_020.cpp(13): >\n<
+//H 12: t_2_020.cpp(14): >int<
+//H 12: t_2_020.cpp(14): > <
+//H 12: t_2_020.cpp(14): >foo<
+//H 12: t_2_020.cpp(14): >;<
+//H 12: t_2_020.cpp(14): >\n<
+//H 12: t_2_020.cpp(15): >#else<
+//H 12: t_2_020.cpp(15): >\n<
+//H 10: t_2_020.cpp(17): #endif
+//H 12: t_2_020.cpp(17): >#endif<
+//H 12: t_2_020.cpp(17): >\n<
+//H 10: t_2_020.cpp(19): #if
+//H 12: t_2_020.cpp(19): > <
+//H 11: t_2_020.cpp(19): #if defined BAR: 0
+//H 12: t_2_020.cpp(19): >\n<
+//H 12: t_2_020.cpp(20): >int<
+//H 12: t_2_020.cpp(20): > <
+//H 12: t_2_020.cpp(20): >bar<
+//H 12: t_2_020.cpp(20): >;<
+//H 12: t_2_020.cpp(20): >\n<
+//H 10: t_2_020.cpp(21): #elif
+//H 12: t_2_020.cpp(21): > <
+//H 11: t_2_020.cpp(21): #elif defined BAR2: 0
+//H 12: t_2_020.cpp(21): >\n<
+//H 12: t_2_020.cpp(22): >float<
+//H 12: t_2_020.cpp(22): > <
+//H 12: t_2_020.cpp(22): >bar<
+//H 12: t_2_020.cpp(22): >;<
+//H 12: t_2_020.cpp(22): >\n<
+//H 12: t_2_020.cpp(23): >#endif<
+//H 12: t_2_020.cpp(23): >\n<
+//H 10: t_2_020.cpp(25): #define
+//H 08: t_2_020.cpp(25): BAZ=
+//H 12: t_2_020.cpp(25): >\n<
+//H 10: t_2_020.cpp(26): #ifdef
+//H 11: t_2_020.cpp(26): #ifdef BAZ: 1
+//H 12: t_2_020.cpp(26): >\n<
+//H 10: t_2_020.cpp(28): #else
+//H 12: t_2_020.cpp(28): >#else<
+//H 12: t_2_020.cpp(28): >\n<
+//H 12: t_2_020.cpp(29): >float<
+//H 12: t_2_020.cpp(29): > <
+//H 12: t_2_020.cpp(29): >baz<
+//H 12: t_2_020.cpp(29): >;<
+//H 12: t_2_020.cpp(29): >\n<
+//H 12: t_2_020.cpp(30): >#endif<
+//H 12: t_2_020.cpp(30): >\n<
+//H 10: t_2_020.cpp(32): #if
+//H 12: t_2_020.cpp(32): > <
+//H 11: t_2_020.cpp(32): #if defined BAZ: 1
+//H 12: t_2_020.cpp(32): >\n<
+//H 10: t_2_020.cpp(34): #elif
+//H 12: t_2_020.cpp(34): > <
+//H 12: t_2_020.cpp(34): >defined<
+//H 12: t_2_020.cpp(34): > <
+//H 12: t_2_020.cpp(34): >BAZ<
+//H 12: t_2_020.cpp(34): >\n<
+//H 12: t_2_020.cpp(35): >float<
+//H 12: t_2_020.cpp(35): > <
+//H 12: t_2_020.cpp(35): >baz1<
+//H 12: t_2_020.cpp(35): >;<
+//H 12: t_2_020.cpp(35): >\n<
+//H 12: t_2_020.cpp(36): >#endif<
+//H 12: t_2_020.cpp(36): >\n<
+//H 10: t_2_020.cpp(38): #ifndef
+//H 11: t_2_020.cpp(38): #ifndef BAZ: 1
+//H 12: t_2_020.cpp(38): >\n<
+//H 12: t_2_020.cpp(39): >int<
+//H 12: t_2_020.cpp(39): > <
+//H 12: t_2_020.cpp(39): >baz2<
+//H 12: t_2_020.cpp(39): >;<
+//H 12: t_2_020.cpp(39): >\n<
+//H 12: t_2_020.cpp(40): >#else<
+//H 12: t_2_020.cpp(40): >\n<
+//H 10: t_2_020.cpp(42): #endif
+//H 12: t_2_020.cpp(42): >#endif<
+//H 12: t_2_020.cpp(42): >\n<
+//H 10: t_2_020.cpp(44): #if
+//H 12: t_2_020.cpp(44): > <
+//H 11: t_2_020.cpp(44): #if 1: 1
+//H 12: t_2_020.cpp(44): >\n<
+//H 10: t_2_020.cpp(46): #elif
+//H 12: t_2_020.cpp(46): > <
+//H 12: t_2_020.cpp(46): >1<
+//H 12: t_2_020.cpp(46): >\n<
+//H 12: t_2_020.cpp(47): >int<
+//H 12: t_2_020.cpp(47): > <
+//H 12: t_2_020.cpp(47): >w2<
+//H 12: t_2_020.cpp(47): >;<
+//H 12: t_2_020.cpp(47): >\n<
+//H 12: t_2_020.cpp(48): >#else<
+//H 12: t_2_020.cpp(48): >\n<
+//H 12: t_2_020.cpp(49): >int<
+//H 12: t_2_020.cpp(49): > <
+//H 12: t_2_020.cpp(49): >w3<
+//H 12: t_2_020.cpp(49): >;<
+//H 12: t_2_020.cpp(49): >\n<
+//H 12: t_2_020.cpp(50): >#endif<
+//H 12: t_2_020.cpp(50): >\n<
+//H 10: t_2_020.cpp(52): #if
+//H 12: t_2_020.cpp(52): > <
+//H 11: t_2_020.cpp(52): #if 0: 0
+//H 12: t_2_020.cpp(52): >\n<
+//H 12: t_2_020.cpp(53): >int<
+//H 12: t_2_020.cpp(53): > <
+//H 12: t_2_020.cpp(53): >x1<
+//H 12: t_2_020.cpp(53): >;<
+//H 12: t_2_020.cpp(53): >\n<
+//H 10: t_2_020.cpp(54): #elif
+//H 12: t_2_020.cpp(54): > <
+//H 11: t_2_020.cpp(54): #elif 1: 1
+//H 12: t_2_020.cpp(54): >\n<
+//H 10: t_2_020.cpp(56): #else
+//H 12: t_2_020.cpp(56): >#else<
+//H 12: t_2_020.cpp(56): >\n<
+//H 12: t_2_020.cpp(57): >int<
+//H 12: t_2_020.cpp(57): > <
+//H 12: t_2_020.cpp(57): >x3<
+//H 12: t_2_020.cpp(57): >;<
+//H 12: t_2_020.cpp(57): >\n<
+//H 12: t_2_020.cpp(58): >#endif<
+//H 12: t_2_020.cpp(58): >\n<
+//H 10: t_2_020.cpp(60): #if
+//H 12: t_2_020.cpp(60): > <
+//H 11: t_2_020.cpp(60): #if 1: 1
+//H 12: t_2_020.cpp(60): >\n<
+//H 10: t_2_020.cpp(62): #elif
+//H 12: t_2_020.cpp(62): > <
+//H 12: t_2_020.cpp(62): >0<
+//H 12: t_2_020.cpp(62): >\n<
+//H 12: t_2_020.cpp(63): >int<
+//H 12: t_2_020.cpp(63): > <
+//H 12: t_2_020.cpp(63): >y2<
+//H 12: t_2_020.cpp(63): >;<
+//H 12: t_2_020.cpp(63): >\n<
+//H 12: t_2_020.cpp(64): >#else<
+//H 12: t_2_020.cpp(64): >\n<
+//H 12: t_2_020.cpp(65): >int<
+//H 12: t_2_020.cpp(65): > <
+//H 12: t_2_020.cpp(65): >y3<
+//H 12: t_2_020.cpp(65): >;<
+//H 12: t_2_020.cpp(65): >\n<
+//H 12: t_2_020.cpp(66): >#endif<
+//H 12: t_2_020.cpp(66): >\n<
+//H 10: t_2_020.cpp(68): #if
+//H 12: t_2_020.cpp(68): > <
+//H 11: t_2_020.cpp(68): #if 0: 0
+//H 12: t_2_020.cpp(68): >\n<
+//H 12: t_2_020.cpp(69): >int<
+//H 12: t_2_020.cpp(69): > <
+//H 12: t_2_020.cpp(69): >z1<
+//H 12: t_2_020.cpp(69): >;<
+//H 12: t_2_020.cpp(69): >\n<
+//H 10: t_2_020.cpp(70): #elif
+//H 12: t_2_020.cpp(70): > <
+//H 11: t_2_020.cpp(70): #elif 0: 0
+//H 12: t_2_020.cpp(70): >\n<
+//H 12: t_2_020.cpp(71): >int<
+//H 12: t_2_020.cpp(71): > <
+//H 12: t_2_020.cpp(71): >z2<
+//H 12: t_2_020.cpp(71): >;<
+//H 12: t_2_020.cpp(71): >\n<
+//H 12: t_2_020.cpp(72): >#else<
+//H 12: t_2_020.cpp(72): >\n<
+//H 10: t_2_020.cpp(74): #endif
+//H 12: t_2_020.cpp(74): >#endif<
+//H 12: t_2_020.cpp(74): >\n<

Modified: trunk/libs/wave/test/testwave/testfiles/test.cfg
==============================================================================
--- trunk/libs/wave/test/testwave/testfiles/test.cfg (original)
+++ trunk/libs/wave/test/testwave/testfiles/test.cfg 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -72,6 +72,7 @@
 t_2_017.cpp
 t_2_018.cpp
 t_2_019.cpp
+t_2_020.cpp
 
 #
 # t_3: Predefined macros

Modified: trunk/libs/wave/test/testwave/testwave_app.cpp
==============================================================================
--- trunk/libs/wave/test/testwave/testwave_app.cpp (original)
+++ trunk/libs/wave/test/testwave/testwave_app.cpp 2010-09-05 18:33:20 EDT (Sun, 05 Sep 2010)
@@ -336,6 +336,7 @@
 #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
         ("noguard,G", "disable include guard detection")
 #endif
+ ("skipped_token_hooks", "record skipped_token hook calls")
     ;
 }
 
@@ -840,6 +841,13 @@
         std::cerr << "initialise_options: initializing options" << std::endl;
     }
 
+ if (vm.count("skipped_token_hooks")) {
+ if (9 == debuglevel) {
+ std::cerr << "initialise_options: option: skipped_token_hooks" << std::endl;
+ }
+ ctx.get_hooks().set_skipped_token_hooks(true);
+ }
+
 // initialize the given context from the parsed options
 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
 // enable C99 mode, if appropriate (implies variadics)


Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk