refactor: fix encoding and move str2bool to shared utils

- Add encoding="utf-8" to open() in tn/main.py and itn/main.py for
  Windows compatibility
- Move str2bool from itn/main.py to tn/utils.py to remove TN's
  dependency on ITN module

Author: pengzhendong

itn/main.py

@@ -14,19 +14,9 @@
 
 import argparse
 
-# TODO(xcsong): multi-language support
 from itn.chinese.inverse_normalizer import InverseNormalizer as ZhInverseNormalizer
 from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer
-
-
-def str2bool(s, default=False):
-    s = s.lower()
-    if s == "true":
-        return True
-    elif s == "false":
-        return False
-    else:
-        return default
+from tn.utils import str2bool
 
 
 def main():
@@ -62,7 +52,7 @@ def main():
         print(normalizer.tag(args.text))
         print(normalizer.normalize(args.text))
     elif args.file:
-        with open(args.file) as fin:
+        with open(args.file, encoding="utf-8") as fin:
             for line in fin:
                 print(normalizer.tag(line.strip()))
                 print(normalizer.normalize(line.strip()))

tn/main.py

@@ -14,7 +14,7 @@
 
 import argparse
 
-from itn.main import str2bool
+from tn.utils import str2bool
 
 # TODO(pzd17 & sxc19): multi-language support
 from tn.chinese.normalizer import Normalizer as ZhNormalizer
@@ -63,7 +63,7 @@ def main():
         print(normalizer.tag(args.text))
         print(normalizer.normalize(args.text))
     elif args.file:
-        with open(args.file) as fin:
+        with open(args.file, encoding="utf-8") as fin:
             for line in fin:
                 print(normalizer.tag(line.strip()))
                 print(normalizer.normalize(line.strip()))

tn/utils.py

@@ -82,3 +82,13 @@ def get_formats(input_f, input_case="cased", is_default=True):
 
     multiple_formats = pynini.string_map(multiple_formats)
     return multiple_formats
+
+
+def str2bool(s, default=False):
+    s = s.lower()
+    if s == "true":
+        return True
+    elif s == "false":
+        return False
+    else:
+        return default