|
|
Boost-Commit : |
Subject: [Boost-commit] svn:boost r76248 - in trunk: boost/spirit/home/karma/detail boost/spirit/home/karma/nonterminal boost/spirit/home/karma/operator boost/spirit/home/lex/qi libs/spirit/doc
From: hartmut.kaiser_at_[hidden]
Date: 2011-12-31 14:39:44
Author: hkaiser
Date: 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
New Revision: 76248
URL: http://svn.boost.org/trac/boost/changeset/76248
Log:
Spirit: improving error reporting from lexer tokens
Text files modified:
trunk/boost/spirit/home/karma/detail/extract_from.hpp | 34 ++++++------
trunk/boost/spirit/home/karma/detail/pass_container.hpp | 56 ++++++++++----------
trunk/boost/spirit/home/karma/nonterminal/rule.hpp | 2
trunk/boost/spirit/home/karma/operator/sequence.hpp | 38 ++++++------
trunk/boost/spirit/home/lex/qi/plain_raw_token.hpp | 16 +++--
trunk/boost/spirit/home/lex/qi/plain_token.hpp | 107 +++++++++++++++++++++++++++++++++++--
trunk/boost/spirit/home/lex/qi/plain_tokenid.hpp | 112 ++++++++++++++++++++++++++++++++++++---
trunk/boost/spirit/home/lex/qi/plain_tokenid_mask.hpp | 20 +++---
trunk/libs/spirit/doc/what_s_new.qbk | 5 +
9 files changed, 292 insertions(+), 98 deletions(-)
Modified: trunk/boost/spirit/home/karma/detail/extract_from.hpp
==============================================================================
--- trunk/boost/spirit/home/karma/detail/extract_from.hpp (original)
+++ trunk/boost/spirit/home/karma/detail/extract_from.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -23,8 +23,8 @@
namespace boost { namespace spirit { namespace traits
{
///////////////////////////////////////////////////////////////////////////
- // This file contains attribute extraction utilities. The utilities
- // provided also accept spirit's unused_type; all no-ops. Compiler
+ // This file contains attribute extraction utilities. The utilities
+ // provided also accept spirit's unused_type; all no-ops. Compiler
// optimization will easily strip these away.
///////////////////////////////////////////////////////////////////////////
@@ -33,13 +33,13 @@
///////////////////////////////////////////////////////////////////////
// extract first and second element of a fusion sequence
template <typename T>
- struct add_const_ref
- : add_reference<typename add_const<T>::type>
+ struct add_const_ref
+ : add_reference<typename add_const<T>::type>
{};
template <typename T, int N>
- struct value_at_c
- : add_const_ref<typename fusion::result_of::value_at_c<T, N>::type>
+ struct value_at_c
+ : add_const_ref<typename fusion::result_of::value_at_c<T, N>::type>
{};
}
@@ -47,7 +47,7 @@
template <typename Attribute, typename Exposed, typename Enable/*= void*/>
struct extract_from_attribute
{
- typedef typename traits::one_element_sequence<Attribute>::type
+ typedef typename traits::one_element_sequence<Attribute>::type
is_one_element_sequence;
typedef typename mpl::eval_if<
@@ -63,7 +63,7 @@
}
// This handles the case where the attribute is a single element fusion
- // sequence. We silently extract the only element and treat it as the
+ // sequence. We silently extract the only element and treat it as the
// attribute to generate output from.
template <typename Context>
static type call(Attribute const& attr, Context& ctx, mpl::true_)
@@ -78,7 +78,7 @@
}
};
- // This handles optional attributes.
+ // This handles optional attributes.
template <typename Attribute, typename Exposed>
struct extract_from_attribute<boost::optional<Attribute>, Exposed>
{
@@ -103,7 +103,7 @@
}
};
- // This handles attributes wrapped inside a boost::ref().
+ // This handles attributes wrapped inside a boost::ref().
template <typename Attribute, typename Exposed>
struct extract_from_attribute<reference_wrapper<Attribute>, Exposed>
{
@@ -120,14 +120,14 @@
template <typename Attribute, typename Exposed, typename Enable>
struct extract_from_container
{
- typedef typename traits::container_value<Attribute const>::type
+ typedef typename traits::container_value<Attribute const>::type
value_type;
- typedef typename is_convertible<value_type, Exposed>::type
+ typedef typename is_convertible<value_type, Exposed>::type
is_convertible_to_value_type;
typedef typename mpl::if_<
mpl::or_<
- is_same<value_type, Exposed>, is_same<Attribute, Exposed> >
+ is_same<value_type, Exposed>, is_same<Attribute, Exposed> >
, Exposed const&, Exposed
>::type type;
@@ -137,7 +137,7 @@
static type call(Attribute const& attr, Context&, mpl::true_, Pred)
{
// return first element from container
- typedef typename traits::container_iterator<Attribute const>::type
+ typedef typename traits::container_iterator<Attribute const>::type
iterator_type;
iterator_type it = traits::begin(attr);
@@ -176,7 +176,7 @@
static type call(Attribute const& attr, Context& ctx)
{
typedef typename mpl::and_<
- traits::is_string<Exposed>, traits::is_string<Attribute>
+ traits::is_string<Exposed>, traits::is_string<Attribute>
>::type handle_strings;
// return first element from container
@@ -208,7 +208,7 @@
return extract_from_attribute<Attribute, Exposed>::call(attr, ctx);
}
- // overload for containers (but not for variants or optionals
+ // overload for containers (but not for variants or optionals
// holding containers)
template <typename Exposed, typename Attribute, typename Context>
inline typename spirit::result_of::extract_from<Exposed, Attribute>::type
@@ -230,7 +230,7 @@
typedef typename mpl::and_<
traits::is_container<Attribute>
, traits::not_is_variant<Attribute>
- , traits::not_is_optional<Attribute>
+ , traits::not_is_optional<Attribute>
>::type is_not_wrapped_container;
return detail::extract_from<Exposed>(attr, ctx
Modified: trunk/boost/spirit/home/karma/detail/pass_container.hpp
==============================================================================
--- trunk/boost/spirit/home/karma/detail/pass_container.hpp (original)
+++ trunk/boost/spirit/home/karma/detail/pass_container.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -38,17 +38,17 @@
Sequence
, typename traits::is_weak_substitute<Attribute, ValueType>::type
, typename mpl::not_<
- traits::is_weak_substitute<Attribute, ValueType>
+ traits::is_weak_substitute<Attribute, ValueType>
>::type>
{};
- // pass_through_container: utility to check decide whether a provided
- // container attribute needs to be passed through to the current component
+ // pass_through_container: utility to check decide whether a provided
+ // container attribute needs to be passed through to the current component
// or of we need to split the container by passing along instances of its
// value type
- // if the expected attribute of the current component is neither a Fusion
- // sequence nor a container, we will pass through the provided container
+ // if the expected attribute of the current component is neither a Fusion
+ // sequence nor a container, we will pass through the provided container
// only if its value type is not compatible with the component
template <typename Container, typename ValueType, typename Attribute
, typename Sequence, typename Enable = void>
@@ -58,9 +58,9 @@
// Specialization for fusion sequences, in this case we check whether all
// the types in the sequence are convertible to the lhs attribute.
- //
+ //
// We return false if the rhs attribute itself is a fusion sequence, which
- // is compatible with the LHS sequence (we want to pass through this
+ // is compatible with the LHS sequence (we want to pass through this
// attribute without it being split apart).
template <typename Container, typename ValueType, typename Attribute
, typename Sequence = mpl::true_>
@@ -71,7 +71,7 @@
{};
// If the value type of the container is not a Fusion sequence, we pass
- // through the container if each of the elements of the Attribute
+ // through the container if each of the elements of the Attribute
// sequence is compatible with either the container or its value type.
template <typename Container, typename ValueType, typename Attribute
, typename Sequence
@@ -87,7 +87,7 @@
};
// If both, the Attribute and the value type of the provided container
- // are Fusion sequences, we pass the container only if the two
+ // are Fusion sequences, we pass the container only if the two
// sequences are not compatible.
template <typename Container, typename ValueType, typename Attribute
, typename Sequence>
@@ -115,14 +115,14 @@
// Specialization for containers
//
// If the value type of the attribute of the current component is not
- // a Fusion sequence, we have to pass through the provided container if
+ // a Fusion sequence, we have to pass through the provided container if
// both are compatible.
template <typename Container, typename ValueType, typename Attribute
, typename Sequence, typename AttributeValueType
, bool IsSequence = fusion::traits::is_sequence<AttributeValueType>::value>
struct pass_through_container_container
: mpl::or_<
- traits::is_weak_substitute<Container, Attribute>
+ traits::is_weak_substitute<Container, Attribute>
, traits::is_weak_substitute<Container, AttributeValueType> >
{};
@@ -148,16 +148,16 @@
// Specialization for exposed optional attributes
//
- // If the type embedded in the exposed optional is not a Fusion
+ // If the type embedded in the exposed optional is not a Fusion
// sequence we pass through the container attribute if it is compatible
- // either to the optionals embedded type or to the containers value
+ // either to the optionals embedded type or to the containers value
// type.
template <typename Container, typename ValueType, typename Attribute
, typename Sequence
, bool IsSequence = fusion::traits::is_sequence<Attribute>::value>
struct pass_through_container_optional
: mpl::or_<
- traits::is_weak_substitute<Container, Attribute>
+ traits::is_weak_substitute<Container, Attribute>
, traits::is_weak_substitute<ValueType, Attribute> >
{};
@@ -175,7 +175,7 @@
template <typename Container, typename ValueType, typename Attribute
, typename Sequence>
struct pass_through_container
- : pass_through_container_base<Container, ValueType, Attribute, Sequence>
+ : pass_through_container_base<Container, ValueType, Attribute, Sequence>
{};
// Handle optional attributes
@@ -184,7 +184,7 @@
struct pass_through_container<
Container, ValueType, boost::optional<Attribute>, Sequence>
: pass_through_container_optional<
- Container, ValueType, Attribute, Sequence>
+ Container, ValueType, Attribute, Sequence>
{};
// If both, the containers value type and the exposed attribute type are
@@ -199,8 +199,8 @@
{};
// Specialization for exposed variant attributes
- //
- // We pass through the container attribute if at least one of the embedded
+ //
+ // We pass through the container attribute if at least one of the embedded
// types in the variant requires to pass through the attribute
#define BOOST_SPIRIT_PASS_THROUGH_CONTAINER(z, N, _) \
@@ -252,7 +252,7 @@
typedef typename F::context_type context_type;
pass_container(F const& f, Iterator begin, Iterator end)
- : f(f), iter(begin), end(end)
+ : f(f), iter(begin), end(end)
{}
bool is_at_end() const
@@ -271,7 +271,7 @@
bool dispatch_container(Component const& component, mpl::false_) const
{
// get the next value to generate from container
- if (!is_at_end() && !f(component, traits::deref(iter)))
+ if (!is_at_end() && !f(component, traits::deref(iter)))
{
// needs to return false as long as everything is ok
traits::next(iter);
@@ -282,8 +282,8 @@
return true;
}
- // this is for the case when the current element is able to handle an
- // attribute which is a container itself, this element will push its
+ // this is for the case when the current element is able to handle an
+ // attribute which is a container itself, this element will push its
// data directly into the attribute container
template <typename Component>
bool dispatch_container(Component const& component, mpl::true_) const
@@ -292,7 +292,7 @@
}
///////////////////////////////////////////////////////////////////////
- // this is for the case when the current element doesn't expect an
+ // this is for the case when the current element doesn't expect an
// attribute
template <typename Component>
bool dispatch_attribute(Component const& component, mpl::false_) const
@@ -305,15 +305,15 @@
bool dispatch_attribute(Component const& component, mpl::true_) const
{
typedef typename traits::container_value<Attr>::type value_type;
- typedef typename
+ typedef typename
traits::attribute_of<Component, context_type>::type
lhs_attribute;
// this predicate detects, whether the value type of the container
- // attribute is a substitute for the attribute of the current
- // element
+ // attribute is a substitute for the attribute of the current
+ // element
typedef mpl::and_<
- traits::handles_container<Component, Attr, context_type>
+ traits::handles_container<Component, Attr, context_type>
, traits::pass_through_container<
Attr, value_type, lhs_attribute, Sequence, karma::domain>
> predicate;
@@ -328,7 +328,7 @@
{
// we need to dispatch depending on the type of the attribute
// of the current element (component). If this is has no attribute
- // we shouldn't use an element of the container but unused_type
+ // we shouldn't use an element of the container but unused_type
// instead
typedef traits::not_is_unused<
typename traits::attribute_of<Component, context_type>::type
Modified: trunk/boost/spirit/home/karma/nonterminal/rule.hpp
==============================================================================
--- trunk/boost/spirit/home/karma/nonterminal/rule.hpp (original)
+++ trunk/boost/spirit/home/karma/nonterminal/rule.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -436,7 +436,7 @@
typename attribute_of<
karma::rule<IteratorA, T1, T2, T3, T4>
, Context, IteratorB
- >::type, Attribute>
+ >::type, Attribute>
{};
}}}
Modified: trunk/boost/spirit/home/karma/operator/sequence.hpp
==============================================================================
--- trunk/boost/spirit/home/karma/operator/sequence.hpp (original)
+++ trunk/boost/spirit/home/karma/operator/sequence.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -138,14 +138,14 @@
OutputIterator, Context, Delimiter> fail_function;
typedef traits::attribute_not_unused<Context> predicate;
- // wrap the attribute in a tuple if it is not a tuple or if the
+ // wrap the attribute in a tuple if it is not a tuple or if the
// attribute of this sequence is a single element tuple
typedef typename attribute<Context>::type_ attr_type_;
typename traits::wrap_if_not_tuple<Attribute
, typename mpl::and_<
traits::one_element_sequence<attr_type_>
, mpl::not_<traits::one_element_sequence<Attribute> >
- >::type
+ >::type
>::type attr(attr_);
// return false if *any* of the generators fail
@@ -155,12 +155,12 @@
typedef typename traits::attribute_size<Attribute>::type size_type;
// fail generating if sequences have not the same (logical) length
- return !r && (!Strict::value ||
- // This ignores container element count (which is not good),
- // but allows valid attributes to succeed. This will lead to
+ return !r && (!Strict::value ||
+ // This ignores container element count (which is not good),
+ // but allows valid attributes to succeed. This will lead to
// false positives (failing generators, even if they shouldn't)
- // if the embedded component is restricting the number of
- // container elements it consumes (i.e. repeat). This solution
+ // if the embedded component is restricting the number of
+ // container elements it consumes (i.e. repeat). This solution
// is not optimal but much better than letting _all_ repetitive
// components fail.
Pred1::value ||
@@ -184,17 +184,17 @@
typename add_const<Attribute>::type
>::type iterator_type;
- typedef
- typename traits::make_indirect_iterator<iterator_type>::type
+ typedef
+ typename traits::make_indirect_iterator<iterator_type>::type
indirect_iterator_type;
typedef detail::pass_container<
- fail_function, Attribute, indirect_iterator_type, mpl::true_>
+ fail_function, Attribute, indirect_iterator_type, mpl::true_>
pass_container;
iterator_type begin = traits::begin(attr_);
iterator_type end = traits::end(attr_);
- pass_container pass(fail_function(sink, ctx, d),
+ pass_container pass(fail_function(sink, ctx, d),
indirect_iterator_type(begin), indirect_iterator_type(end));
bool r = fusion::any(elements, pass);
@@ -210,11 +210,11 @@
bool generate(OutputIterator& sink, Context& ctx, Delimiter const& d
, Attribute const& attr) const
{
- typedef typename traits::is_container<Attribute>::type
+ typedef typename traits::is_container<Attribute>::type
is_container;
typedef typename attribute<Context>::type_ attr_type_;
- typedef typename traits::one_element_sequence<attr_type_>::type
+ typedef typename traits::one_element_sequence<attr_type_>::type
is_one_element_sequence;
return generate_impl(sink, ctx, d, attr, is_container()
@@ -234,7 +234,7 @@
};
template <typename Elements>
- struct sequence
+ struct sequence
: base_sequence<Elements, mpl::false_, sequence<Elements> >
{
typedef base_sequence<Elements, mpl::false_, sequence> base_sequence_;
@@ -244,10 +244,10 @@
};
template <typename Elements>
- struct strict_sequence
+ struct strict_sequence
: base_sequence<Elements, mpl::true_, strict_sequence<Elements> >
{
- typedef base_sequence<Elements, mpl::true_, strict_sequence>
+ typedef base_sequence<Elements, mpl::true_, strict_sequence>
base_sequence_;
strict_sequence(Elements const& subject)
@@ -260,12 +260,12 @@
namespace detail
{
template <typename Elements, bool strict_mode = false>
- struct make_sequence
+ struct make_sequence
: make_nary_composite<Elements, sequence>
{};
template <typename Elements>
- struct make_sequence<Elements, true>
+ struct make_sequence<Elements, true>
: make_nary_composite<Elements, strict_sequence>
{};
}
@@ -283,7 +283,7 @@
{
typedef iterator_range<detail::indirect_iterator<Iterator> > type;
};
-}}}
+}}}
namespace boost { namespace spirit { namespace traits
{
Modified: trunk/boost/spirit/home/lex/qi/plain_raw_token.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/qi/plain_raw_token.hpp (original)
+++ trunk/boost/spirit/home/lex/qi/plain_raw_token.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -1,6 +1,6 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(BOOST_SPIRIT_LEX_PLAIN_RAW_TOKEN_JUN_03_2011_0853PM)
@@ -25,6 +25,7 @@
#include <boost/mpl/or.hpp>
#include <boost/type_traits/is_integral.hpp>
#include <boost/type_traits/is_enum.hpp>
+#include <boost/lexical_cast.hpp>
namespace boost { namespace spirit
{
@@ -59,7 +60,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename TokenId>
- struct plain_raw_token
+ struct plain_raw_token
: primitive_parser<plain_raw_token<TokenId> >
{
template <typename Context, typename Iterator>
@@ -80,11 +81,11 @@
qi::skip_over(first, last, skipper); // always do a pre-skip
if (first != last) {
- // simply match the token id with the id this component has
+ // simply match the token id with the id this component has
// been initialized with
- typedef typename
- boost::detail::iterator_traits<Iterator>::value_type
+ typedef typename
+ boost::detail::iterator_traits<Iterator>::value_type
token_type;
typedef typename token_type::id_type id_type;
@@ -101,7 +102,8 @@
template <typename Context>
info what(Context& /*context*/) const
{
- return info("token");
+ return info("raw_token",
+ "raw_token(" + boost::lexical_cast<utf8_string>(id) + ")");
}
TokenId id;
Modified: trunk/boost/spirit/home/lex/qi/plain_token.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/qi/plain_token.hpp (original)
+++ trunk/boost/spirit/home/lex/qi/plain_token.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -1,6 +1,6 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(BOOST_SPIRIT_LEX_PLAIN_TOKEN_NOV_11_2007_0451PM)
@@ -23,8 +23,10 @@
#include <boost/fusion/include/vector.hpp>
#include <boost/fusion/include/at.hpp>
#include <boost/mpl/or.hpp>
+#include <boost/mpl/and.hpp>
#include <boost/type_traits/is_integral.hpp>
#include <boost/type_traits/is_enum.hpp>
+#include <boost/lexical_cast.hpp>
namespace boost { namespace spirit
{
@@ -43,12 +45,26 @@
, terminal_ex<tag::token, fusion::vector1<A0> >
> : mpl::or_<is_integral<A0>, is_enum<A0> > {};
+ // enables token(idmin, idmax)
+ template <typename A0, typename A1>
+ struct use_terminal<qi::domain
+ , terminal_ex<tag::token, fusion::vector2<A0, A1> >
+ > : mpl::and_<
+ mpl::or_<is_integral<A0>, is_enum<A0> >
+ , mpl::or_<is_integral<A1>, is_enum<A1> >
+ > {};
+
// enables *lazy* token(id)
template <>
struct use_lazy_terminal<
qi::domain, tag::token, 1
> : mpl::true_ {};
+ // enables *lazy* token(idmin, idmax)
+ template <>
+ struct use_lazy_terminal<
+ qi::domain, tag::token, 2
+ > : mpl::true_ {};
}}
namespace boost { namespace spirit { namespace qi
@@ -60,7 +76,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename TokenId>
- struct plain_token
+ struct plain_token
: primitive_parser<plain_token<TokenId> >
{
template <typename Context, typename Iterator>
@@ -82,11 +98,11 @@
qi::skip_over(first, last, skipper); // always do a pre-skip
if (first != last) {
- // simply match the token id with the id this component has
+ // simply match the token id with the id this component has
// been initialized with
- typedef typename
- boost::detail::iterator_traits<Iterator>::value_type
+ typedef typename
+ boost::detail::iterator_traits<Iterator>::value_type
token_type;
typedef typename token_type::id_type id_type;
@@ -103,13 +119,71 @@
template <typename Context>
info what(Context& /*context*/) const
{
- return info("token");
+ return info("token",
+ "token(" + boost::lexical_cast<utf8_string>(id) + ")");
}
TokenId id;
};
///////////////////////////////////////////////////////////////////////////
+ template <typename TokenId>
+ struct plain_token_range
+ : primitive_parser<plain_token_range<TokenId> >
+ {
+ template <typename Context, typename Iterator>
+ struct attribute
+ {
+ typedef typename Iterator::base_iterator_type iterator_type;
+ typedef iterator_range<iterator_type> type;
+ };
+
+ plain_token_range(TokenId const& idmin, TokenId const& idmax)
+ : idmin(idmin), idmax(idmax) {}
+
+ template <typename Iterator, typename Context
+ , typename Skipper, typename Attribute>
+ bool parse(Iterator& first, Iterator const& last
+ , Context& /*context*/, Skipper const& skipper
+ , Attribute& attr) const
+ {
+ qi::skip_over(first, last, skipper); // always do a pre-skip
+
+ if (first != last) {
+ // simply match the token id with the id this component has
+ // been initialized with
+
+ typedef typename
+ boost::detail::iterator_traits<Iterator>::value_type
+ token_type;
+ typedef typename token_type::id_type id_type;
+
+ token_type const& t = *first;
+ if (id_type(idmin) >= t.id() && id_type(idmin) <= t.id())
+ {
+ spirit::traits::assign_to(t, attr);
+ ++first;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ template <typename Context>
+ info what(Context& /*context*/) const
+ {
+ return info("token_range"
+ , "token(" +
+ boost::lexical_cast<utf8_string>(idmin) + ", " +
+ boost::lexical_cast<utf8_string>(idmax) + ")"
+ );
+ return info("token_range");
+ }
+
+ TokenId idmin, idmax;
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
// Parser generators: make_xxx function (objects)
///////////////////////////////////////////////////////////////////////////
template <typename Modifiers>
@@ -135,6 +209,20 @@
return result_type(fusion::at_c<0>(term.args));
}
};
+
+ template <typename Modifiers, typename TokenId>
+ struct make_primitive<terminal_ex<tag::token, fusion::vector2<TokenId, TokenId> >
+ , Modifiers>
+ {
+ typedef plain_token_range<TokenId> result_type;
+
+ template <typename Terminal>
+ result_type operator()(Terminal const& term, unused_type) const
+ {
+ return result_type(fusion::at_c<0>(term.args)
+ , fusion::at_c<1>(term.args));
+ }
+ };
}}}
namespace boost { namespace spirit { namespace traits
@@ -144,6 +232,11 @@
struct handles_container<qi::plain_token<Idtype>, Attr, Context, Iterator>
: mpl::true_
{};
+
+ template<typename Idtype, typename Attr, typename Context, typename Iterator>
+ struct handles_container<qi::plain_token_range<Idtype>, Attr, Context, Iterator>
+ : mpl::true_
+ {};
}}}
#endif
Modified: trunk/boost/spirit/home/lex/qi/plain_tokenid.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/qi/plain_tokenid.hpp (original)
+++ trunk/boost/spirit/home/lex/qi/plain_tokenid.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -1,6 +1,6 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(BOOST_SPIRIT_LEX_PLAIN_TOKENID_NOV_26_2010_0944AM)
@@ -23,8 +23,10 @@
#include <boost/fusion/include/vector.hpp>
#include <boost/fusion/include/at.hpp>
#include <boost/mpl/or.hpp>
+#include <boost/mpl/and.hpp>
#include <boost/type_traits/is_integral.hpp>
#include <boost/type_traits/is_enum.hpp>
+#include <boost/lexical_cast.hpp>
namespace boost { namespace spirit
{
@@ -43,11 +45,26 @@
, terminal_ex<tag::tokenid, fusion::vector1<A0> >
> : mpl::or_<is_integral<A0>, is_enum<A0> > {};
+ // enables tokenid(idmin, idmax)
+ template <typename A0, typename A1>
+ struct use_terminal<qi::domain
+ , terminal_ex<tag::tokenid, fusion::vector2<A0, A1> >
+ > : mpl::and_<
+ mpl::or_<is_integral<A0>, is_enum<A0> >
+ , mpl::or_<is_integral<A1>, is_enum<A1> >
+ > {};
+
// enables *lazy* tokenid(id)
template <>
struct use_lazy_terminal<
qi::domain, tag::tokenid, 1
> : mpl::true_ {};
+
+ // enables *lazy* tokenid(idmin, idmax)
+ template <>
+ struct use_lazy_terminal<
+ qi::domain, tag::tokenid, 2
+ > : mpl::true_ {};
}}
namespace boost { namespace spirit { namespace qi
@@ -58,11 +75,11 @@
using spirit::tokenid_type;
///////////////////////////////////////////////////////////////////////////
- // The plain_tokenid represents a simple token defined by the lexer inside
- // a Qi grammar. The difference to plain_token is that it exposes the
+ // The plain_tokenid represents a simple token defined by the lexer inside
+ // a Qi grammar. The difference to plain_token is that it exposes the
// matched token id instead of the iterator_range of the matched input.
template <typename TokenId>
- struct plain_tokenid
+ struct plain_tokenid
: primitive_parser<plain_tokenid<TokenId> >
{
template <typename Context, typename Iterator>
@@ -83,11 +100,11 @@
qi::skip_over(first, last, skipper); // always do a pre-skip
if (first != last) {
- // simply match the token id with the id this component has
+ // simply match the token id with the id this component has
// been initialized with
- typedef typename
- boost::detail::iterator_traits<Iterator>::value_type
+ typedef typename
+ boost::detail::iterator_traits<Iterator>::value_type
token_type;
typedef typename token_type::id_type id_type;
@@ -104,17 +121,73 @@
template <typename Context>
info what(Context& /*context*/) const
{
- return info("tokenid");
+ return info("tokenid",
+ "tokenid(" + boost::lexical_cast<utf8_string>(id) + ")");
}
TokenId id;
};
///////////////////////////////////////////////////////////////////////////
+ template <typename TokenId>
+ struct plain_tokenid_range
+ : primitive_parser<plain_tokenid_range<TokenId> >
+ {
+ template <typename Context, typename Iterator>
+ struct attribute
+ {
+ typedef TokenId type;
+ };
+
+ plain_tokenid_range(TokenId const& idmin, TokenId const& idmax)
+ : idmin(idmin), idmax(idmax) {}
+
+ template <typename Iterator, typename Context
+ , typename Skipper, typename Attribute>
+ bool parse(Iterator& first, Iterator const& last
+ , Context& /*context*/, Skipper const& skipper
+ , Attribute& attr) const
+ {
+ qi::skip_over(first, last, skipper); // always do a pre-skip
+
+ if (first != last) {
+ // simply match the token id with the id this component has
+ // been initialized with
+
+ typedef typename
+ boost::detail::iterator_traits<Iterator>::value_type
+ token_type;
+ typedef typename token_type::id_type id_type;
+
+ token_type const& t = *first;
+ if (id_type(idmin) >= t.id() && id_type(idmin) <= t.id())
+ {
+ spirit::traits::assign_to(id, attr);
+ ++first;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ template <typename Context>
+ info what(Context& /*context*/) const
+ {
+ return info("tokenid_range"
+ , "token(" +
+ boost::lexical_cast<utf8_string>(idmin) + ", " +
+ boost::lexical_cast<utf8_string>(idmax) + ")"
+ );
+ }
+
+ TokenId idmin, idmax;
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
// Parser generators: make_xxx function (objects)
///////////////////////////////////////////////////////////////////////////
template <typename Modifiers>
- struct make_primitive<tag::tokenid, Modifiers>
+ struct make_primitive<tag::tokenid, Modifiers>
{
typedef plain_tokenid<std::size_t> result_type;
@@ -136,6 +209,20 @@
return result_type(fusion::at_c<0>(term.args));
}
};
+
+ template <typename Modifiers, typename TokenId>
+ struct make_primitive<terminal_ex<tag::tokenid, fusion::vector2<TokenId, TokenId> >
+ , Modifiers>
+ {
+ typedef plain_tokenid_range<TokenId> result_type;
+
+ template <typename Terminal>
+ result_type operator()(Terminal const& term, unused_type) const
+ {
+ return result_type(fusion::at_c<0>(term.args)
+ , fusion::at_c<1>(term.args));
+ }
+ };
}}}
namespace boost { namespace spirit { namespace traits
@@ -145,6 +232,11 @@
struct handles_container<qi::plain_tokenid<Idtype>, Attr, Context, Iterator>
: mpl::true_
{};
+
+ template<typename Idtype, typename Attr, typename Context, typename Iterator>
+ struct handles_container<qi::plain_tokenid_range<Idtype>, Attr, Context, Iterator>
+ : mpl::true_
+ {};
}}}
#endif
Modified: trunk/boost/spirit/home/lex/qi/plain_tokenid_mask.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/qi/plain_tokenid_mask.hpp (original)
+++ trunk/boost/spirit/home/lex/qi/plain_tokenid_mask.hpp 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -1,6 +1,6 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(BOOST_SPIRIT_LEX_PLAIN_TOKENID_MASK_JUN_03_2011_0929PM)
@@ -25,6 +25,7 @@
#include <boost/mpl/or.hpp>
#include <boost/type_traits/is_integral.hpp>
#include <boost/type_traits/is_enum.hpp>
+#include <boost/lexical_cast.hpp>
namespace boost { namespace spirit
{
@@ -53,8 +54,8 @@
using spirit::tokenid_mask_type;
///////////////////////////////////////////////////////////////////////////
- // The plain_tokenid represents a simple token defined by the lexer inside
- // a Qi grammar. The difference to plain_token is that it exposes the
+ // The plain_tokenid represents a simple token defined by the lexer inside
+ // a Qi grammar. The difference to plain_token is that it exposes the
// matched token id instead of the iterator_range of the matched input.
// Additionally it applies the given mask to the matched token id.
template <typename Mask>
@@ -79,16 +80,16 @@
qi::skip_over(first, last, skipper); // always do a pre-skip
if (first != last) {
- // simply match the token id with the mask this component has
+ // simply match the token id with the mask this component has
// been initialized with
- typedef typename
- boost::detail::iterator_traits<Iterator>::value_type
+ typedef typename
+ boost::detail::iterator_traits<Iterator>::value_type
token_type;
typedef typename token_type::id_type id_type;
token_type const& t = *first;
- if ((t.id() & mask) == id_type(mask))
+ if ((t.id() & mask) == id_type(mask))
{
spirit::traits::assign_to(t.id(), attr);
++first;
@@ -101,7 +102,8 @@
template <typename Context>
info what(Context& /*context*/) const
{
- return info("tokenid_mask");
+ return info("tokenid_mask",
+ "tokenid_mask(" + boost::lexical_cast<utf8_string>(mask) + ")");
}
Mask mask;
Modified: trunk/libs/spirit/doc/what_s_new.qbk
==============================================================================
--- trunk/libs/spirit/doc/what_s_new.qbk (original)
+++ trunk/libs/spirit/doc/what_s_new.qbk 2011-12-31 14:39:42 EST (Sat, 31 Dec 2011)
@@ -15,6 +15,11 @@
* Integrated Vitaly Budovski's patch to add binary floating point parsers and
generators.
+* Fixed #6253: `lex::lexertl::generate_static_dfa` compiler errors if lexer has
+ wchar_t as underlying stream type.
+* Added the possibility to match a token range when parsing with a lexer by
+ using `qi::token(min, max)`, or `qi::tokenid(min, max)`. Both parsers match
+ any token with a token id in the range [min, max].
[endsect]
Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk