more UnicodeNormalizer optimalizations

Author: kosloot

include/frog/mblem_mod.h

@@ -68,6 +68,7 @@ class Mblem {
   void makeUnique();
   void add_lemmas( const std::vector<folia::Word*>&,
 		   const frog_data& ) const;
+  TiCC::UnicodeNormalizer& normalizer(){ return _normalizer; };
  private:
   icu::UnicodeString call_server( const icu::UnicodeString& );
   void read_transtable( const std::string& );

include/frog/mbma_mod.h

@@ -80,6 +80,7 @@ class Mbma {
   static std::string mbma_tagset;
   static std::string pos_tagset;
   static std::string clex_tagset;
+  TiCC::UnicodeNormalizer& normalizer(){ return _normalizer; };
   Mbma( const Mbma& ) = delete;
   Mbma& operator=( const Mbma& ) = delete;
  private:

include/frog/mwu_chunker_mod.h

@@ -97,6 +97,7 @@ class Mwu {
   std::string mwu_tagset;
   icu::UnicodeString glue_tag;
   TiCC::UniFilter *filter;
+  TiCC::UnicodeNormalizer _normalizer;
   Mwu( const Mwu& ) = delete; // no copies
   Mwu operator=( const Mwu& ) = delete; // no copies
 };

src/mblem_mod.cxx

@@ -98,7 +98,7 @@ bool Mblem::fill_ts_map( const string& file ){
     return false;
   }
   UnicodeString line;
-  while ( TiCC::getline( is, line ) ){
+  while ( TiCC::getline( is, _normalizer, line ) ){
     if ( line.isEmpty() || line[0] == '#' ){
       continue;
     }

src/mblem_prog.cxx

@@ -174,7 +174,7 @@ bool init(){
 
 void Test( istream& in, ostream& os ){
   UnicodeString line;
-  while ( TiCC::getline( in, line ) ){
+  while ( TiCC::getline( in, myMblem.normalizer(),line ) ){
     if ( line.isEmpty() ) {
       os << endl;
       continue;

src/mbma_mod.cxx

@@ -131,7 +131,7 @@ void Mbma::init_cgn( const string& main, const string& sub ) {
   ifstream tc( main );
   if ( tc ){
     UnicodeString line;
-    while ( TiCC::getline( tc, line) ) {
+    while ( TiCC::getline( tc, _normalizer, line) ) {
       vector<UnicodeString> tmp = TiCC::split_at( line, " " );
       if ( tmp.size() < 2 ){
 	LOG << "splitting '" << line << "' failed" << endl;
@@ -146,7 +146,7 @@ void Mbma::init_cgn( const string& main, const string& sub ) {
   ifstream tc1( sub );
   if ( tc1 ){
     UnicodeString line;
-    while( TiCC::getline( tc1, line ) ) {
+    while( TiCC::getline( tc1, _normalizer, line ) ) {
       vector<UnicodeString> tmp = TiCC::split_at( line, " " );
       if ( tmp.size() == 2 ){
 	TAGconv.insert( make_pair( tmp[0], tmp[1] ) );

src/mbma_prog.cxx

@@ -183,7 +183,7 @@ bool check_next( const UnicodeString& tag ){
 
 void Test( istream& in ){
   UnicodeString line;
-  while ( TiCC::getline( in, line ) ){
+  while ( TiCC::getline( in, myMbma.normalizer(), line ) ){
     line = line.trim();
     if ( line.isEmpty() ){
       continue;

src/mwu_chunker_mod.cxx

@@ -136,7 +136,7 @@ bool Mwu::read_mwus( const string& fname) {
     return false;
   }
   UnicodeString line;
-  while( TiCC::getline( mwufile, line ) ) {
+  while( TiCC::getline( mwufile, _normalizer, line ) ) {
     vector<UnicodeString> res1 = TiCC::split_at(line, " ");
     if ( res1.size() == 2 ){
       vector<UnicodeString> res2 = TiCC::split_at(res1[0], "_");;
@@ -211,7 +211,7 @@ bool Mwu::init( const TiCC::Configuration& config ) {
     glue_tag = "SPEC(deeleigen)";
   }
   else {
-    glue_tag = TiCC::UnicodeFromUTF8(val);
+    glue_tag = TiCC::UnicodeFromUTF8(val,_normalizer);
   }
 
   string cls = config.lookUp( "outputclass" );

src/ner_tagger_mod.cxx

@@ -125,7 +125,7 @@ bool NERTagger::fill_ners( const string& cat,
   int long_err_cnt = 0;
   size_t ner_cnt = 0;
   UnicodeString line;
-  while ( TiCC::getline( is, line ) ){
+  while ( TiCC::getline( is, _normalizer, line ) ){
     if ( line.isEmpty() || line[0] == '#' ){
       continue;
     }

src/tagger_base.cxx

@@ -91,7 +91,7 @@ bool BaseTagger::fill_map( const string& file ){
     return false;
   }
   UnicodeString line;
-  while( TiCC::getline( is, line ) ){
+  while( TiCC::getline( is, _normalizer, line ) ){
     if ( line.isEmpty() || line[0] == '#' ){
       continue;
     }