From faff751faa5b65cd91fa1971901c4dccc360e522 Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Tue, 9 Jun 2026 19:35:50 +0800 Subject: [PATCH 1/2] feat: support nbest output in normalize/tag (#296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalize(input, nbest=1) returns a single string (default, backward compatible). normalize(input, nbest=N) returns a list of N-best results. Example: n.normalize('海淀区108号', nbest=3) # ['海淀区一百零八号', '海淀区十八号', '海淀区幺零八号'] --- tn/processor.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tn/processor.py b/tn/processor.py index faabb07..a125b67 100644 --- a/tn/processor.py +++ b/tn/processor.py @@ -104,21 +104,29 @@ def build_fst(self, prefix, cache_dir, overwrite_cache): logger.info("fst path: {}".format(tagger_path)) logger.info(" {}".format(verbalizer_path)) - def tag(self, input): + def tag(self, input, nbest=1): if len(input) == 0: - return "" + return "" if nbest == 1 else [""] input = escape(input) lattice = input @ self.tagger - return shortestpath(lattice, nshortest=1, unique=True).string() + if nbest == 1: + return shortestpath(lattice, nshortest=1, unique=True).string() + lattice = shortestpath(lattice.project("output").rmepsilon(), nshortest=nbest, unique=True) + paths = lattice.paths() + results = [] + while not paths.done(): + results.append(paths.ostring()) + paths.next() + return results def verbalize(self, input): - # Only words from the blacklist are contained. if len(input) == 0: return "" output = TokenParser(self.ordertype).reorder(input) - # We need escape for pynini to build the fst from string. lattice = escape(output) @ self.verbalizer return shortestpath(lattice, nshortest=1, unique=True).string() - def normalize(self, input): - return self.verbalize(self.tag(input)) + def normalize(self, input, nbest=1): + if nbest == 1: + return self.verbalize(self.tag(input)) + return [self.verbalize(tagged) for tagged in self.tag(input, nbest)] From e57b889bdd8c896175d2f149d803210df873c8fa Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Tue, 9 Jun 2026 19:53:15 +0800 Subject: [PATCH 2/2] fix: only use digit-by-digit reading for 4-digit years (#302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously 2-digit numbers followed by 年/年度/赛季 were read digit-by-digit (38年 => 三八年). Now only 4-digit years use this pattern, so 38年 => 三十八年 (via number) while 2024年 => 二零二四年. --- tn/chinese/rules/measure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tn/chinese/rules/measure.py b/tn/chinese/rules/measure.py index f5ecb4a..cac5bf1 100644 --- a/tn/chinese/rules/measure.py +++ b/tn/chinese/rules/measure.py @@ -47,12 +47,12 @@ def build_tagger(self): measure @= self.build_rule(cross("两" + unit, "二" + unit), l="[BOS]") measure @= self.build_rule(cross("到两" + unit, "到二" + unit), r="[EOS]") - # -xxxx年, -xx年 + # xxxx年, xxxx-xxxx年 digits = self.cardinal.digits - cardinal = digits**2 | digits**4 + yyyy = digits**4 unit = accep("年") | accep("年度") | accep("赛季") - prefix = cardinal + (rmspace + unit).ques + to - annual = prefix.ques + cardinal + unit + prefix = yyyy + (rmspace + unit).ques + to + annual = prefix.ques + yyyy + unit tagger = insert('value: "') + (measure | annual) + insert('"')