|
|
Boost-Commit : |
Subject: [Boost-commit] svn:boost r62749 - trunk/boost
From: andrey.semashev_at_[hidden]
Date: 2010-06-10 13:13:18
Author: andysem
Date: 2010-06-10 13:13:14 EDT (Thu, 10 Jun 2010)
New Revision: 62749
URL: http://svn.boost.org/trac/boost/changeset/62749
Log:
Fixed tabs, removed unneeded spaces. Silenced a few warnings on MSVC and made the code compilable if no cwctype header is available.
Text files modified:
trunk/boost/token_functions.hpp | 174 +++++++++++++++++++--------------------
1 files changed, 86 insertions(+), 88 deletions(-)
Modified: trunk/boost/token_functions.hpp
==============================================================================
--- trunk/boost/token_functions.hpp (original)
+++ trunk/boost/token_functions.hpp 2010-06-10 13:13:14 EDT (Thu, 10 Jun 2010)
@@ -35,12 +35,14 @@
#include <stdexcept>
#include <string>
#include <cctype>
-#include <cwctype>
#include <algorithm> // for find_if
#include <boost/config.hpp>
#include <boost/assert.hpp>
#include <boost/detail/workaround.hpp>
#include <boost/mpl/if.hpp>
+#if !defined(BOOST_NO_CWCTYPE)
+#include <cwctype>
+#endif
//
// the following must not be macros if we are to prefix them
@@ -65,8 +67,10 @@
namespace std{
using ::ispunct;
using ::isspace;
+#if !defined(BOOST_NO_CWCTYPE)
using ::iswpunct;
using ::iswspace;
+#endif
}
#endif
@@ -84,15 +88,15 @@
struct escaped_list_error : public std::runtime_error{
escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
};
-
+
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
// MSVC does not like the following typename
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
- template <class Char,
+ template <class Char,
class Traits = typename std::basic_string<Char>::traits_type >
#else
- template <class Char,
+ template <class Char,
class Traits = std::basic_string<Char>::traits_type >
#endif
class escaped_list_separator {
@@ -108,7 +112,7 @@
};
string_type escape_;
string_type c_;
- string_type quote_;
+ string_type quote_;
bool last_;
bool is_escape(Char e) {
@@ -148,21 +152,21 @@
}
public:
-
+
explicit escaped_list_separator(Char e = '\\',
Char c = ',',Char q = '\"')
: escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
-
+
escaped_list_separator(string_type e, string_type c, string_type q)
: escape_(e), c_(c), quote_(q), last_(false) { }
-
+
void reset() {last_=false;}
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next,InputIterator end,Token& tok) {
bool bInQuote = false;
tok = Token();
-
+
if (next == end) {
if (last_) {
last_ = false;
@@ -201,24 +205,42 @@
//===========================================================================
// The classes here are used by offset_separator and char_separator to implement
// faster assigning of tokens using assign instead of +=
-
+
namespace tokenizer_detail {
//===========================================================================
// Tokenizer was broken for wide character separators, at least on Windows, since
- // CRT functions isspace etc are only expect values in [0, 0xFF]. Debug build asserts
+ // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
// if higher values are passed in. The traits extension class should take care of this.
// Assuming that the conditional will always get optimized out in the function
// implementations, argument types are not a problem since both forms of character classifiers
// expect an int.
+ // In case there is no cwctype header, we implement the checks manually.
+ // We make use of the fact that the tested categories should fit in ASCII.
template<typename traits>
struct traits_extension : public traits {
typedef typename traits::char_type char_type;
- static bool isspace(char_type c) {
- return sizeof(char_type) == 1 ? std::isspace(c) : std::iswspace(c);
+ static bool isspace(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ if (sizeof(char_type) == 1)
+ return std::isspace(c) != 0;
+ else
+ return std::iswspace(c) != 0;
+#else
+ return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
+#endif
}
- static bool ispunct(char_type c) {
- return sizeof(char_type) == 1 ? std::ispunct(c) : std::iswpunct(c);
+ static bool ispunct(char_type c)
+ {
+#if !defined(BOOST_NO_CWCTYPE)
+ if (sizeof(char_type) == 1)
+ return std::ispunct(c) != 0;
+ else
+ return std::iswpunct(c) != 0;
+#else
+ return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
+#endif
}
};
@@ -255,26 +277,20 @@
}
- template<class Token, class Value>
- static void plus_equal(Token &, const Value &) {
-
- }
+ template<class Token, class Value>
+ static void plus_equal(Token &, const Value &) { }
// If we are doing an assign, there is no need for the
- // the clear.
+ // the clear.
//
template<class Token>
- static void clear(Token &) {
-
- }
+ static void clear(Token &) { }
};
template <>
struct assign_or_plus_equal<std::input_iterator_tag> {
template<class Iterator, class Token>
- static void assign(Iterator b, Iterator e, Token &t) {
-
- }
+ static void assign(Iterator b, Iterator e, Token &t) { }
template<class Token, class Value>
static void plus_equal(Token &t, const Value &v) {
t += v;
@@ -310,10 +326,10 @@
typedef typename cat::type iterator_category;
};
-
-}
-
+ } // namespace tokenizer_detail
+
+
//===========================================================================
// The offset_separator class, which is a model of TokenizerFunction.
// Offset breaks a string into tokens based on a range of offsets
@@ -325,7 +341,7 @@
unsigned int current_offset_;
bool wrap_offsets_;
bool return_partial_last_;
-
+
public:
template <typename Iter>
offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
@@ -333,7 +349,7 @@
: offsets_(begin,end), current_offset_(0),
wrap_offsets_(wrap_offsets),
return_partial_last_(return_partial_last) { }
-
+
offset_separator()
: offsets_(1,1), current_offset_(),
wrap_offsets_(true), return_partial_last_(true) { }
@@ -346,18 +362,16 @@
bool operator()(InputIterator& next, InputIterator end, Token& tok)
{
typedef tokenizer_detail::assign_or_plus_equal<
-#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
- typename
-#endif
- tokenizer_detail::get_iterator_category<
- InputIterator>::iterator_category> assigner;
-
+ BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
+ InputIterator
+ >::iterator_category
+ > assigner;
BOOST_ASSERT(!offsets_.empty());
-
+
assigner::clear(tok);
InputIterator start(next);
-
+
if (next == end)
return false;
@@ -368,7 +382,7 @@
else
return false;
}
-
+
int c = offsets_[current_offset_];
int i = 0;
for (; i < c; ++i) {
@@ -376,11 +390,11 @@
assigner::plus_equal(tok,*next++);
}
assigner::assign(start,next,tok);
-
+
if (!return_partial_last_)
if (i < (c-1) )
return false;
-
+
++current_offset_;
return true;
}
@@ -404,16 +418,11 @@
enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
-#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
- template <typename Char,
- typename Tr = typename std::basic_string<Char>::traits_type >
-#else
- template <typename Char,
- typename Tr = std::basic_string<Char>::traits_type >
-#endif
+ template <typename Char,
+ typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
class char_separator
{
- typedef tokenizer_detail::traits_extension<Tr> Traits;
+ typedef tokenizer_detail::traits_extension<Tr> Traits;
typedef std::basic_string<Char,Traits> string_type;
public:
explicit
@@ -434,8 +443,8 @@
// use ispunct() for kept delimiters and isspace for dropped.
explicit
char_separator()
- : m_use_ispunct(true),
- m_use_isspace(true),
+ : m_use_ispunct(true),
+ m_use_isspace(true),
m_empty_tokens(drop_empty_tokens) { }
void reset() { }
@@ -444,11 +453,10 @@
bool operator()(InputIterator& next, InputIterator end, Token& tok)
{
typedef tokenizer_detail::assign_or_plus_equal<
-#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
- typename
-#endif
- tokenizer_detail::get_iterator_category<
- InputIterator>::iterator_category> assigner;
+ BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
+ InputIterator
+ >::iterator_category
+ > assigner;
assigner::clear(tok);
@@ -456,7 +464,7 @@
if (m_empty_tokens == drop_empty_tokens)
for (; next != end && is_dropped(*next); ++next)
{ }
-
+
InputIterator start(next);
if (m_empty_tokens == drop_empty_tokens) {
@@ -473,13 +481,13 @@
// append all the non delim characters
for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
assigner::plus_equal(tok,*next);
- }
+ }
else { // m_empty_tokens == keep_empty_tokens
-
+
// Handle empty token at the end
if (next == end)
{
- if (m_output_done == false)
+ if (m_output_done == false)
{
m_output_done = true;
assigner::assign(start,next,tok);
@@ -488,7 +496,7 @@
else
return false;
}
-
+
if (is_kept(*next)) {
if (m_output_done == false)
m_output_done = true;
@@ -520,9 +528,9 @@
bool m_use_isspace;
empty_token_policy m_empty_tokens;
bool m_output_done;
-
+
bool is_kept(Char E) const
- {
+ {
if (m_kept_delims.length())
return m_kept_delims.find(E) != string_type::npos;
else if (m_use_ispunct) {
@@ -552,26 +560,21 @@
// cannot be returned as tokens. These are often whitespace
// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
-#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
- template <class Char,
- class Tr = typename std::basic_string<Char>::traits_type >
-#else
- template <class Char,
- class Tr = std::basic_string<Char>::traits_type >
-#endif
+ template <class Char,
+ class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
class char_delimiters_separator {
- private:
+ private:
- typedef tokenizer_detail::traits_extension<Tr> Traits;
+ typedef tokenizer_detail::traits_extension<Tr> Traits;
typedef std::basic_string<Char,Traits> string_type;
string_type returnable_;
string_type nonreturnable_;
bool return_delims_;
bool no_ispunct_;
bool no_isspace_;
-
+
bool is_ret(Char E)const
- {
+ {
if (returnable_.length())
return returnable_.find(E) != string_type::npos;
else{
@@ -594,7 +597,7 @@
}
}
}
-
+
public:
explicit char_delimiters_separator(bool return_delims = false,
const Char* returnable = 0,
@@ -603,7 +606,7 @@
nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
return_delims_(return_delims), no_ispunct_(returnable!=0),
no_isspace_(nonreturnable!=0) { }
-
+
void reset() { }
public:
@@ -611,16 +614,16 @@
template <typename InputIterator, typename Token>
bool operator()(InputIterator& next, InputIterator end,Token& tok) {
tok = Token();
-
+
// skip past all nonreturnable delims
// skip past the returnable only if we are not returning delims
for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
&& !return_delims_ ) );++next) { }
-
+
if (next == end) {
return false;
}
-
+
// if we are to return delims and we are one a returnable one
// move past it and stop
if (is_ret(*next) && return_delims_) {
@@ -631,8 +634,8 @@
// append all the non delim characters
for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
tok+=*next;
-
-
+
+
return true;
}
};
@@ -641,9 +644,4 @@
} //namespace boost
-#endif
-
-
-
-
-
+#endif
Boost-Commit list run by bdawes at acm.org, david.abrahams at rcn.com, gregod at cs.rpi.edu, cpdaniel at pacbell.net, john at johnmaddock.co.uk