[Boost-commit] svn:boost r42776 - in trunk: boost/xpressive libs/xpressive/test

From: eric_at_[hidden]
Date: 2008-01-14 15:26:59

Author: eric_niebler
Date: 2008-01-14 15:26:58 EST (Mon, 14 Jan 2008)
New Revision: 42776
URL: http://svn.boost.org/trac/boost/changeset/42776

Log:
add skip(), for specifying a skip regex
Added:
   trunk/libs/xpressive/test/test_skip.cpp   (contents, props changed)
Text files modified: 
   trunk/boost/xpressive/regex_primitives.hpp |   129 ++++++++++++++++++++++++++++++++++++++++
   trunk/libs/xpressive/test/Jamfile.v2       |     1                                         
   2 files changed, 130 insertions(+), 0 deletions(-)

Modified: trunk/boost/xpressive/regex_primitives.hpp
==============================================================================
--- trunk/boost/xpressive/regex_primitives.hpp	(original)
+++ trunk/boost/xpressive/regex_primitives.hpp	2008-01-14 15:26:58 EST (Mon, 14 Jan 2008)
@@ -115,6 +115,93 @@
     #undef minus_one
     #endif
 
+    // replace "Expr" with "keep(*State) >> Expr"
+    struct skip_primitives : proto::callable
+    {
+        template<typename Sig>
+        struct result;
+
+        template<typename This, typename Expr, typename State, typename Visitor>
+        struct result<This(Expr, State, Visitor)>
+          : proto::shift_right<
+                typename proto::unary_expr<
+                    keeper_tag
+                  , typename proto::dereference<State>::type
+                >::type
+              , Expr
+            >
+        {};
+
+        template<typename Expr, typename State, typename Visitor>
+        typename result<void(Expr, State, Visitor)>::type
+        operator ()(Expr const &expr, State const &state, Visitor &visitor) const
+        {
+            typedef typename result<void(Expr, State, Visitor)>::type type;
+            type that = {{{state}}, expr};
+            return that;
+        }
+    };
+
+    struct Primitives
+      : proto::or_<
+            proto::terminal<proto::_>
+          , proto::comma<proto::_, proto::_>
+          , proto::subscript<proto::terminal<set_initializer>, proto::_> 
+          , proto::assign<proto::terminal<set_initializer>, proto::_>
+          , proto::assign<proto::terminal<attribute_placeholder<proto::_> >, proto::_>
+          , proto::complement<Primitives>
+        >
+    {};
+
+    struct SkipGrammar
+      : proto::or_<
+            proto::when<Primitives, skip_primitives>
+          , proto::assign<proto::terminal<mark_placeholder>, SkipGrammar>   // don't "skip" mark tags
+          , proto::subscript<SkipGrammar, proto::_>                         // don't put skips in actions
+          , proto::binary_expr<modifier_tag, proto::_, SkipGrammar>         // don't skip modifiers
+          , proto::unary_expr<lookbehind_tag, proto::_>                     // don't skip lookbehinds
+          , proto::nary_expr<proto::_, proto::vararg<SkipGrammar> >         // everything else is fair game!
+        >
+    {};
+
+    template<typename Skip>
+    struct skip_directive
+    {
+        typedef typename proto::result_of::as_expr<Skip>::type skip_type;
+
+        skip_directive(Skip const &skip)
+          : skip_(proto::as_expr(skip))
+        {}
+
+        template<typename Sig>
+        struct result;
+
+        template<typename This, typename Expr>
+        struct result<This(Expr)>
+          : proto::shift_right<
+                typename SkipGrammar::result<void(
+                    typename proto::result_of::as_expr<Expr>::type
+                  , skip_type
+                  , mpl::void_
+                )>::type
+              , typename proto::dereference<skip_type>::type
+            >
+        {};
+
+        template<typename Expr>
+        typename result<skip_directive(Expr)>::type
+        operator ()(Expr const &expr) const
+        {
+            mpl::void_ ignore;
+            typedef typename result<skip_directive(Expr)>::type result_type;
+            result_type result = {SkipGrammar()(proto::as_expr(expr), this->skip_, ignore), {skip_}};
+            return result;
+        }
+
+    private:
+        skip_type skip_;
+    };
+
 /*
 ///////////////////////////////////////////////////////////////////////////////
 /// INTERNAL ONLY
@@ -637,6 +724,48 @@
 proto::terminal<detail::attribute_placeholder<mpl::int_<8> > >::type const a8 = {{}};
 proto::terminal<detail::attribute_placeholder<mpl::int_<9> > >::type const a9 = {{}};
 
+///////////////////////////////////////////////////////////////////////////////
+/// \brief Specify which characters to skip when matching a regex.
+///
+/// <tt>skip()</tt> instructs the regex engine to skip certain characters when matching
+/// a regex. It is most useful for writing regexes that ignore whitespace.
+/// For instance, the following specifies a regex that skips whitespace and
+/// punctuation:
+///
+/// \code
+/// // A sentence is one or more words separated by whitespace
+/// // and punctuation.
+/// sregex word = +alpha;
+/// sregex sentence = skip(set[_s | punct])( +word );
+/// \endcode
+///
+/// The way it works in the above example is to insert
+/// <tt>keep(*set[_s | punct])</tt> before each primitive within the regex.
+/// A "primitive" includes terminals like strings, character sets and nested
+/// regexes. A final <tt>*set[_s | punct]</tt> is added to the end of the
+/// regex. The regex <tt>sentence</tt> specified above is equivalent to
+/// the following:
+///
+/// \code
+/// sregex sentence = +( keep(*set[_s | punct]) >> word )
+///                        >> *set[_s | punct];
+/// \endcode
+///
+/// \attention Skipping does not affect how nested regexes are handles because
+/// they are treated atomically. String literals are also treated
+/// atomically; that is, no skipping is done within a string literal. So
+/// <tt>skip(_s)("this that")</tt> is not the same as
+/// <tt>skip(_s)("this" >> as_xpr("that"))</tt>. The first will only match
+/// when there is only one space between "this" and "that". The second will
+/// skip any and all whitespace between "this" and "that".
+///
+/// \param skip A regex that specifies which characters to skip.
+template<typename Skip>
+detail::skip_directive<Skip> skip(Skip const &skip)
+{
+    return detail::skip_directive<Skip>(skip);
+}
+
 namespace detail
 {
     inline void ignore_unused_regex_primitives()

Modified: trunk/libs/xpressive/test/Jamfile.v2
==============================================================================
--- trunk/libs/xpressive/test/Jamfile.v2	(original)
+++ trunk/libs/xpressive/test/Jamfile.v2	2008-01-14 15:26:58 EST (Mon, 14 Jan 2008)
@@ -57,6 +57,7 @@
          [ run test_symbols.cpp ]
          [ run test_dynamic.cpp ]
          [ run test_dynamic_grammar.cpp ]
+         [ run test_skip.cpp ]
          [ link multiple_defs1.cpp multiple_defs2.cpp : : multiple_defs ]
          [ compile test_basic_regex.cpp ]
          [ compile test_match_results.cpp ]

Added: trunk/libs/xpressive/test/test_skip.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/xpressive/test/test_skip.cpp	2008-01-14 15:26:58 EST (Mon, 14 Jan 2008)
@@ -0,0 +1,96 @@
+///////////////////////////////////////////////////////////////////////////////
+// test_skip.hpp
+//
+//  Copyright 2004 Eric Niebler. Distributed under the Boost
+//  Software License, Version 1.0. (See accompanying file
+//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <map>
+#include <iostream>
+#include <boost/xpressive/xpressive.hpp>
+#include <boost/xpressive/regex_actions.hpp>
+#include <boost/test/unit_test.hpp>
+
+using namespace boost::unit_test;
+using namespace boost::xpressive;
+
+void test1()
+{
+    std::string s = "a a b b c c";
+
+    sregex rx =
+        "a a" >>
+        skip(_s)
+        (
+             (s1= as_xpr('b')) >>
+             as_xpr('b') >>
+            *as_xpr('c')    // causes backtracking
+        ) >> 
+        "c c";
+
+    smatch what;
+    BOOST_CHECK( regex_match(s, what, rx) );
+
+    s = "123,456,789";
+    sregex rx2 = skip(',')(+_d);
+    BOOST_CHECK( regex_match(s, what, rx2) );
+
+    s = "foo";
+    sregex rx3 = skip(_s)(after("fo") >> 'o');
+    BOOST_CHECK( regex_search(s, what, rx3) );
+}
+
+template<typename Expr>
+void test_skip_aux(Expr const &expr)
+{
+    sregex rx = skip(_s)(expr);
+}
+
+void test_skip()
+{
+    int i=0;
+    std::map<std::string, int> syms;
+    std::locale loc;
+
+    test_skip_aux( 'a' );
+    test_skip_aux( _ );
+    test_skip_aux( +_ );
+    test_skip_aux( -+_ );
+    test_skip_aux( !_ );
+    test_skip_aux( -!_ );
+    test_skip_aux( repeat<0,42>(_) );
+    test_skip_aux( -repeat<0,42>(_) );
+    test_skip_aux( _ >> 'a' );
+    test_skip_aux( _ >> 'a' | _ );
+    test_skip_aux( _ >> 'a' | _ >> 'b' );
+    test_skip_aux( s1= _ >> 'a' | _ >> 'b' );
+    test_skip_aux( icase(_ >> 'a' | _ >> 'b') );
+    test_skip_aux( imbue(loc)(_ >> 'a' | _ >> 'b') );
+    test_skip_aux( (set='a') );
+    test_skip_aux( (set='a','b') );
+    test_skip_aux( ~(set='a') );
+    test_skip_aux( ~(set='a','b') );
+    test_skip_aux( range('a','b') );
+    test_skip_aux( ~range('a','b') );
+    test_skip_aux( set['a' | alpha] );
+    test_skip_aux( ~set['a' | alpha] );
+    test_skip_aux( before(_) );
+    test_skip_aux( ~before(_) );
+    test_skip_aux( after(_) );
+    test_skip_aux( ~after(_) );
+    test_skip_aux( keep(*_) );
+    test_skip_aux( (*_)[ref(i) = as<int>(_) + 1] );
+    test_skip_aux( (a1= syms)[ref(i) = a1 + 1] );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// init_unit_test_suite
+//
+test_suite* init_unit_test_suite( int argc, char* argv[] )
+{
+    test_suite *test = BOOST_TEST_SUITE("test skip()");
+
+    test->add(BOOST_TEST_CASE(&test1));
+
+    return test;
+}