added optional normalization parameters to UnicodeFromUTF8() and UnicodeToUTF8()

The default is to always produce NFC.

Author: kosloot

include/ticcutils/Unicode.h

@@ -45,14 +45,14 @@
 namespace TiCC {
   using namespace icu;
 
-  std::string UnicodeToUTF8( const UnicodeString&  );
+  std::string UnicodeToUTF8( const UnicodeString&,
+			     const std::string& = "" );
 
   UnicodeString UnicodeFromEnc( const std::string& ,
 				const std::string& = "UTF8" );
 
-  inline UnicodeString UnicodeFromUTF8( const std::string& s ){
-    return UnicodeString::fromUTF8( s );
-  }
+  UnicodeString UnicodeFromUTF8( const std::string&,
+				 const std::string& = "" );
 
   /// \brief a class that can normalize UnicodeStrings to NFC/NFD/NFKC/NFKD
   class UnicodeNormalizer {

src/Unicode.cxx

@@ -50,17 +50,28 @@ namespace TiCC {
     return UnicodeString( s.c_str(), s.length(), enc.c_str() );
   }
 
-  string UnicodeToUTF8( const UnicodeString& s ){
+  string UnicodeToUTF8( const UnicodeString& s,
+			const string& normalization ){
     /// convert a UnicodeString to a UTF-8 string
     /*!
       \param s the UnicodeString to convert
+      \param normalization the normalization to use. Default NFC
       \return an UTF-8 encoded string
     */
+    UnicodeNormalizer UN( normalization);
+    UnicodeString normalized = UN.normalize( s );
     string result;
-    s.toUTF8String(result);
+    normalized.toUTF8String(result);
     return result;
   }
 
+  UnicodeString UnicodeFromUTF8( const string& s,
+				 const string& normalization ){
+    UnicodeNormalizer UN( normalization);
+    UnicodeString result = UnicodeString::fromUTF8( s );
+    return UN.normalize( result );
+  }
+
   UnicodeNormalizer::UnicodeNormalizer( const string& enc ): _normalizer(0) {
     /// create an UnicodeNormalizer object
     /*!

src/runtest.cxx

@@ -889,13 +889,14 @@ void test_unicode( const string& path ){
   UnicodeString ng1 = N.normalize( greek1 );
   UnicodeString ng2 = N.normalize( greek2 );
   assertEqual( UnicodeToUTF8(ng1), UnicodeToUTF8(ng2) );
+  assertEqual( UnicodeToUTF8(ng1,"NFD"), UnicodeToUTF8(ng2,"NFD") );
   N.setMode("NFD");
   UnicodeString ng11 = N.normalize( greek1 );
   UnicodeString ng12 = N.normalize( greek2 );
   assertEqual( UnicodeToUTF8(ng11), UnicodeToUTF8(ng12) );
   string utf8_1 = "ἀντιϰειμένου";
-  string utf8_2 = "ἀντικειμένου";
-  assertEqual( TiCC::utf8_uppercase( utf8_1 ), "ἈΝΤΙΚΕΙΜΈΝΟΥ" );
+  string utf8_2 = "ἀντικειμένου";
+  assertEqual( TiCC::utf8_uppercase( utf8_1 ), "ἈΝΤΙΚΕΙΜΈΝΟΥ" );
   assertEqual( TiCC::utf8_lowercase( "ἈΝΤΙΚΕΙΜΈΝΟΥ" ), utf8_2 );
   assertEqual( TiCC::utf8_uppercase( "æ en ß en œ" ), "Æ EN SS EN Œ" );
 }