use new functions from ticcutils Unicode.h for small speedup and

smaller memory footprint

Author: kosloot

include/ucto/tokenize.h

@@ -381,7 +381,7 @@ namespace Tokenizer {
 
     void appendText( folia::FoliaElement * ) const;
 
-    TiCC::UnicodeNormalizer normalizer;
+    mutable TiCC::UnicodeNormalizer normalizer;
     std::string inputEncoding;
 
     const UnicodeString& detect_type( UChar32 );

src/setting.cxx

@@ -292,7 +292,7 @@ namespace Tokenizer {
     }
     else {
       UnicodeString line;
-      while ( TiCC::getline( f, line ) ){
+      while ( TiCC::getline( f, _normalizer, line ) ){
 	line.trim();
 	if ((line.length() > 0) && (line[0] != '#')) {
 	  if ( tokDebug >= 5 ){
@@ -329,7 +329,7 @@ namespace Tokenizer {
     }
     else {
       UnicodeString line;
-      while ( TiCC::getline( f, line ) ){
+      while ( TiCC::getline( f, _normalizer, line ) ){
 	line.trim();
 	if ((line.length() > 0) && (line[0] != '#')) {
 	  if ( tokDebug >= 5 ){
@@ -370,7 +370,7 @@ namespace Tokenizer {
     }
     else {
       UnicodeString line;
-      while ( TiCC::getline( f, line ) ){
+      while ( TiCC::getline( f, _normalizer, line ) ){
 	line.trim();
 	if ((line.length() > 0) && (line[0] != '#')) {
 	  if ( tokDebug >= 5 ){
@@ -431,7 +431,7 @@ namespace Tokenizer {
     }
     else {
       UnicodeString line;
-      while ( TiCC::getline( f, line ) ){
+      while ( TiCC::getline( f, _normalizer, line ) ){
 	line.trim();
 	if ((line.length() > 0) && (line[0] != '#')) {
 	  if ( tokDebug >= 5 ){
@@ -650,7 +650,7 @@ namespace Tokenizer {
       }
       int rule_count = 0;
       UnicodeString line;
-      while ( TiCC::getline( f, line ) ){
+      while ( TiCC::getline( f, _normalizer, line ) ){
 	if ( line.indexOf( "%include" ) != -1 ){
 	  UnicodeString file = UnicodeString(line, 9 );
 	  switch ( mode ){
@@ -850,7 +850,7 @@ namespace Tokenizer {
       if ( !add_tokens.empty() ){
 	ifstream adt( add_tokens );
 	UnicodeString line;
-	while ( TiCC::getline( adt, line ) ){
+	while ( TiCC::getline( adt, _normalizer, line ) ){
 	  UnicodeString entry = escape_regex( line );
 	  if ( !entry.isEmpty() ){
 	    if ( !patterns[TOKENS].isEmpty() ){

src/tokenize.cxx

@@ -720,7 +720,7 @@ namespace Tokenizer {
 	args["name"] = s.second->set_file;
 	args["generate_id"] = "next()";
 	args["type"] = "datasource";
-	args["version"] = TiCC::UnicodeToUTF8(s.second->version);
+	args["version"] = TiCC::UnicodeToUTF8(s.second->version,normalizer);
 	doc->add_processor( args, data_proc );
 	args.clear();
 	args["processor"] = proc->id();
@@ -1368,7 +1368,7 @@ namespace Tokenizer {
       if ( !ids.empty() ){
 	args["generate_id"] = ids;
       }
-      args["class"] = TiCC::UnicodeToUTF8(tok.type);
+      args["class"] = TiCC::UnicodeToUTF8(tok.type,normalizer);
       if ( tok.role & NOSPACE ){
 	args["space"] = "no";
       }
@@ -1528,7 +1528,7 @@ namespace Tokenizer {
       // New elements
       folia::KWargs args;
       args["xml:id"] = orig->generateId( "tokenized" );
-      args["class"] = TiCC::UnicodeToUTF8(tok.type);
+      args["class"] = TiCC::UnicodeToUTF8(tok.type,normalizer);
       if ( tok.role & NOSPACE ){
 	args["space"] = "no";
       }
@@ -3583,7 +3583,7 @@ namespace Tokenizer {
     }
     else {
       set_file = it->second->set_file;
-      version = TiCC::UnicodeToUTF8(it->second->version);
+      version = TiCC::UnicodeToUTF8(it->second->version,normalizer);
       return true;
     }
   }