[Boost-commit] svn:boost r73170 - in trunk/libs/locale: src/encoding test

Subject: [Boost-commit] svn:boost r73170 - in trunk/libs/locale: src/encoding test
From: artyomtnk_at_[hidden]
Date: 2011-07-17 07:04:58

Author: artyom
Date: 2011-07-17 07:04:57 EDT (Sun, 17 Jul 2011)
New Revision: 73170
URL: http://svn.boost.org/trac/boost/changeset/73170

Log:
- Fixed correct error report in case Windows codepage is not
  supported
- Updated tests to skip situations where particular code pages
  may not be supported in Windows

Text files modified: 
   trunk/libs/locale/src/encoding/wconv_codepage.ipp |   102 ++++++++++++++++++++++----------------- 
   trunk/libs/locale/test/test_codepage.cpp          |    23 +++++++-                                
   trunk/libs/locale/test/test_message.cpp           |    24 ++++++++                                
   3 files changed, 99 insertions(+), 50 deletions(-)

Modified: trunk/libs/locale/src/encoding/wconv_codepage.ipp
==============================================================================
--- trunk/libs/locale/src/encoding/wconv_codepage.ipp	(original)
+++ trunk/libs/locale/src/encoding/wconv_codepage.ipp	2011-07-17 07:04:57 EDT (Sun, 17 Jul 2011)
@@ -31,6 +31,7 @@
     struct windows_encoding {
         char const *name;
         unsigned codepage;
+        unsigned was_tested;
     };
 
     bool operator<(windows_encoding const &l,windows_encoding const &r)
@@ -39,50 +40,50 @@
     }
 
     windows_encoding all_windows_encodings[] = {
-        { "big5",       950 },
-        { "cp1250",     1250 },
-        { "cp1251",     1251 },
-        { "cp1252",     1252 },
-        { "cp1253",     1253 },
-        { "cp1254",     1254 },
-        { "cp1255",     1255 },
-        { "cp1256",     1256 },
-        { "cp1257",     1257 },
-        { "cp874",      874 },
-        { "cp932",      932 },
-        { "eucjp",      20932 },
-        { "euckr",      51949 },
-        { "gb18030",    54936 },
-        { "gb2312",     936 },
-        { "iso2022jp",  50220 },
-        { "iso2022kr",  50225 },
-        { "iso88591",   28591 },
-        { "iso885913",  28603 },
-        { "iso885915",  28605 },
-        { "iso88592",   28592 },
-        { "iso88593",   28593 },
-        { "iso88594",   28594 },
-        { "iso88595",   28595 },
-        { "iso88596",   28596 },
-        { "iso88597",   28597 },
-        { "iso88598",   28598 },
-        { "iso88599",   28599 },
-        { "koi8r",      20866 },
-        { "koi8u",      21866 },
-        { "shiftjis",   932 },
-        { "sjis",       932 },
-        { "usascii",    20127 },
-        { "utf8",       65001 },
-        { "windows1250",        1250 },
-        { "windows1251",        1251 },
-        { "windows1252",        1252 },
-        { "windows1253",        1253 },
-        { "windows1254",        1254 },
-        { "windows1255",        1255 },
-        { "windows1256",        1256 },
-        { "windows1257",        1257 },
-        { "windows874",         874 },
-        { "windows932",         932 },
+        { "big5",       950, 0 },
+        { "cp1250",     1250, 0 },
+        { "cp1251",     1251, 0 },
+        { "cp1252",     1252, 0 },
+        { "cp1253",     1253, 0 },
+        { "cp1254",     1254, 0 },
+        { "cp1255",     1255, 0 },
+        { "cp1256",     1256, 0 },
+        { "cp1257",     1257, 0 },
+        { "cp874",      874, 0 },
+        { "cp932",      932, 0 },
+        { "eucjp",      20932, 0 },
+        { "euckr",      51949, 0 },
+        { "gb18030",    54936, 0 },
+        { "gb2312",     936, 0 },
+        { "iso2022jp",  50220, 0 },
+        { "iso2022kr",  50225, 0 },
+        { "iso88591",   28591, 0 },
+        { "iso885913",  28603, 0 },
+        { "iso885915",  28605, 0 },
+        { "iso88592",   28592, 0 },
+        { "iso88593",   28593, 0 },
+        { "iso88594",   28594, 0 },
+        { "iso88595",   28595, 0 },
+        { "iso88596",   28596, 0 },
+        { "iso88597",   28597, 0 },
+        { "iso88598",   28598, 0 },
+        { "iso88599",   28599, 0 },
+        { "koi8r",      20866, 0 },
+        { "koi8u",      21866, 0 },
+        { "shiftjis",   932, 0 },
+        { "sjis",       932, 0 },
+        { "usascii",    20127, 0 },
+        { "utf8",       65001, 0 },
+        { "windows1250",        1250, 0 },
+        { "windows1251",        1251, 0 },
+        { "windows1252",        1252, 0 },
+        { "windows1253",        1253, 0 },
+        { "windows1254",        1254, 0 },
+        { "windows1255",        1255, 0 },
+        { "windows1256",        1256, 0 },
+        { "windows1257",        1257, 0 },
+        { "windows874",         874, 0 },
+        { "windows932",         932, 0 },
     };
 
     size_t remove_substitutions(std::vector<wchar_t> &v)
@@ -181,7 +182,18 @@
         windows_encoding *end = all_windows_encodings + n;
         windows_encoding *ptr = std::lower_bound(begin,end,ref);
         if(ptr!=end && strcmp(ptr->name,charset.c_str())==0) {
-            return ptr->codepage;
+            if(ptr->was_tested) {
+                return ptr->codepage;
+            }
+            else if(IsValidCodePage(ptr->codepage)) {
+                // the thread safety is not an issue, maximum
+                // it would be checked more then once
+                ptr->was_tested=1;
+                return ptr->codepage;
+            }
+            else {
+                return -1;
+            }
         }
         return -1;
         

Modified: trunk/libs/locale/test/test_codepage.cpp
==============================================================================
--- trunk/libs/locale/test/test_codepage.cpp	(original)
+++ trunk/libs/locale/test/test_codepage.cpp	2011-07-17 07:04:57 EDT (Sun, 17 Jul 2011)
@@ -10,6 +10,7 @@
 #include <boost/locale/generator.hpp>
 #include <boost/locale/localization_backend.hpp>
 #include <boost/locale/info.hpp>
+#include <boost/locale/config.hpp>
 #include <fstream>
 #include "test_locale.hpp"
 #include "test_locale_tools.hpp"
@@ -22,7 +23,16 @@
 # include <locale.h>
 #endif
 
+#if !defined(BOOST_LOCALE_WITH_ICU) && !defined(BOOST_LOCALE_WITH_ICONV) && (defined(BOOST_WINDOWS) || defined(__CYGWIN__))
+#ifndef NOMINMAX
+# define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+
+
 bool test_iso;
+bool test_iso_8859_8 = true;
 bool test_utf;
 bool test_sjis;
 
@@ -125,8 +135,10 @@
     }
     
     if(test_iso) {
-        std::cout << "    ISO8859-8" << std::endl;
-        test_ok<Char>("hello \xf9\xec\xe5\xed",g(he_il_8bit),to<Char>("hello שלום"));
+        if(test_iso_8859_8) {
+            std::cout << "    ISO8859-8" << std::endl;
+            test_ok<Char>("hello \xf9\xec\xe5\xed",g(he_il_8bit),to<Char>("hello שלום"));
+        }
         std::cout << "    ISO8859-1" << std::endl;
         test_ok<Char>(to<char>("grüße\nn i"),g(en_us_8bit),to<Char>("grüße\nn i"));
         test_wfail<Char>("grüßen שלום",g(en_us_8bit),7);
@@ -302,7 +314,8 @@
 void test_to()
 {
     test_pos<Char>(to<char>("grüßen"),utf<Char>("grüßen"),"ISO8859-1");
-    test_pos<Char>("\xf9\xec\xe5\xed",utf<Char>("שלום"),"ISO8859-8");
+    if(test_iso_8859_8)
+        test_pos<Char>("\xf9\xec\xe5\xed",utf<Char>("שלום"),"ISO8859-8");
     test_pos<Char>("grüßen",utf<Char>("grüßen"),"UTF-8");
     test_pos<Char>("abc\"\xf0\xa0\x82\x8a\"",utf<Char>("abc\"\xf0\xa0\x82\x8a\""),"UTF-8");
     
@@ -329,6 +342,10 @@
         #ifndef BOOST_LOCALE_NO_POSIX_BACKEND
         def.push_back("posix");
         #endif
+
+        #if !defined(BOOST_LOCALE_WITH_ICU) && !defined(BOOST_LOCALE_WITH_ICONV) && (defined(BOOST_WINDOWS) || defined(__CYGWIN__))
+        test_iso_8859_8 = IsValidCodePage(28598)!=0;
+        #endif
         
         
         for(int type = 0; type < int(def.size()); type ++ ) {

Modified: trunk/libs/locale/test/test_message.cpp
==============================================================================
--- trunk/libs/locale/test/test_message.cpp	(original)
+++ trunk/libs/locale/test/test_message.cpp	2011-07-17 07:04:57 EDT (Sun, 17 Jul 2011)
@@ -306,6 +306,7 @@
     #endif
 }
 
+bool iso_8859_8_not_supported = false;
 
 
 int main(int argc,char **argv)
@@ -348,7 +349,21 @@
             std::string locales[] = { "he_IL.UTF-8", "he_IL.ISO8859-8" };
 
             for(unsigned i=0;i<sizeof(locales)/sizeof(locales[0]);i++){
-                std::locale l=g(locales[i]);
+                std::locale l;
+                
+                if(i==1) {
+                    try {
+                        l = g(locales[i]);
+                    }
+                    catch(boost::locale::conv::invalid_charset_error const &e) {
+                        std::cout << "Looks like ISO-8859-8 is not supported! skipping" << std::endl;
+                        iso_8859_8_not_supported = true;
+                        continue;
+                    }
+                }
+                else {
+                        l = g(locales[i]);
+                }
                 
                 std::cout << "  Testing "<<locales[i]<<std::endl;
                 std::cout << "    single forms" << std::endl;
@@ -435,8 +450,13 @@
             TEST(file_loader_is_actually_called);
             TEST(bl::translate("hello").str(l)=="שלום");
         }
-        std::cout << "Testing non-US-ASCII keys" << std::endl; 
+        if(iso_8859_8_not_supported)
+        {
+            std::cout << "ISO 8859-8 not supported so skipping non-US-ASCII keys" << std::endl; 
+        }
+        else 
         {
+            std::cout << "Testing non-US-ASCII keys" << std::endl; 
             std::cout << "  UTF-8 keys" << std::endl; 
             {
                 boost::locale::generator g;