Refactor Korean TN cardinal and postprocessing logic based on review feedback

Signed-off-by: Jinwoo Bae <bbae7050@gmail.com>

Author: bbae0312

nemo_text_processing/text_normalization/ko/data/number/teen.tsv

@@ -1,3 +1,4 @@
+1	십
 2	이십
 3	삼십
 4	사십

nemo_text_processing/text_normalization/ko/data/number/ty.tsv

@@ -23,7 +23,7 @@
 from pynini.export import export
 from pynini.lib import byte, pynutil, utf8
 
-from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels
+from nemo_text_processing.text_normalization.en.utils import load_labels
 from nemo_text_processing.utils.logging import logger
 
 NEMO_CHAR = utf8.VALID_UTF8_CHAR

nemo_text_processing/text_normalization/ko/graph_utils.py

@@ -18,10 +18,7 @@
 from pynini.lib import pynutil
 
 from nemo_text_processing.text_normalization.ko.graph_utils import (
-    NEMO_WHITE_SPACE,
     GraphFst,
-    delete_extra_space,
-    delete_space,
     generator_main,
 )
 

nemo_text_processing/text_normalization/ko/taggers/cardinal.py

@@ -18,9 +18,7 @@
 import pynini
 
 from nemo_text_processing.text_normalization.en.graph_utils import (
-    NEMO_NOT_SPACE,
     NEMO_SIGMA,
-    delete_space,
     generator_main,
 )
 from nemo_text_processing.utils.logging import logger
@@ -41,73 +39,15 @@ def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
         far_file = None
         if cache_dir is not None and cache_dir != "None":
             os.makedirs(cache_dir, exist_ok=True)
-            far_file = os.path.join(cache_dir, "zh_tn_post_processing.far")
+            far_file = os.path.join(cache_dir, "ko_tn_post_processing.far")
         if not overwrite_cache and far_file and os.path.exists(far_file):
             self.fst = pynini.Far(far_file, mode="r")["post_process_graph"]
             logger.info(f'Post processing graph was restored from {far_file}.')
         else:
-            self.set_punct_dict()
-            self.fst = self.get_punct_postprocess_graph()
+            self.fst = self.get_postprocess_graph()
 
             if far_file:
                 generator_main(far_file, {"post_process_graph": self.fst})
-
-    def set_punct_dict(self):
-        self.punct_marks = {
-            "'": [
-                "'",
-                '´',
-                'ʹ',
-                'ʻ',
-                'ʼ',
-                'ʽ',
-                'ʾ',
-                'ˈ',
-                'ˊ',
-                'ˋ',
-                '˴',
-                'ʹ',
-                '΄',
-                '՚',
-                '՝',
-                'י',
-                '׳',
-                'ߴ',
-                'ߵ',
-                'ᑊ',
-                'ᛌ',
-                '᾽',
-                '᾿',
-                '`',
-                '´',
-                '῾',
-                '‘',
-                '’',
-                '‛',
-                '′',
-                '‵',
-                'ꞌ',
-                '＇',
-                '｀',
-                '𖽑',
-                '𖽒',
-            ],
-        }
-
-    def get_punct_postprocess_graph(self):
-        """
-        Returns graph to post process punctuation marks.
-
-        {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept.
-        By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks.
-        """
-
-        remove_space_around_single_quote = pynini.cdrewrite(
-            delete_space, NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA)
-        )
-        # this works if spaces in between (good)
-        # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA
-
-        graph = remove_space_around_single_quote.optimize()
-
-        return graph
+        
+    def get_postprocess_graph(self):
+        return pynini.cdrewrite(pynini.cross("", ""), "", "", pynini.closure(NEMO_SIGMA)).optimize()

nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pynini
-
 from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst
 from nemo_text_processing.text_normalization.ko.verbalizers.cardinal import CardinalFst
 

nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py

@@ -16,4 +16,33 @@
 9999999999999~구조구천구백구십구억구천구백구십구만구천구백구십구
 99999999999999~구십구조구천구백구십구억구천구백구십구만구천구백구십구
 999999999999999~구백구십구조구천구백구십구억구천구백구십구만구천구백구십구
-9999999999999999~구천구백구십구조구천구백구십구억구천구백구십구만구천구백구십구
+9999999999999999~구천구백구십구조구천구백구십구억구천구백구십구만구천구백구십구
+19~십구
+76~칠십육
+379~삼백칠십구
+850~팔백오십
+1004~천사
+8326~팔천삼백이십육
+10383~만삼백팔십삼
+34892~삼만사천팔백구십이
+573234~오십칠만삼천이백삼십사
+982010~구십팔만이천십
+2349023~이백삼십사만구천이십삼
+4303189~사백삼십만삼천백팔십구
+60321589~육천삼십이만천오백팔십구
+88234568~팔천팔백이십삼만사천오백육십팔
+792133923~칠억구천이백십삼만삼천구백이십삼
+187624689~일억팔천칠백육십이만사천육백팔십구
+2304050708~이십삼억사백오만칠백팔
+6436789729~육십사억삼천육백칠십팔만구천칠백이십구
+78234580257~칠백팔십이억삼천사백오십팔만이백오십칠
+987654321345~구천팔백칠십육억오천사백삼십이만천삼백사십오
+2345678901234~이조삼천사백오십육억칠천팔백구십만천이백삼십사
+35791357913579~삼십오조칠천구백십삼억오천칠백구십일만삼천오백칠십구
+470369258147036~사백칠십조삼천육백구십이억오천팔백십사만칠천삼십육
+5048258149517395~오천사십팔조이천오백팔십일억사천구백오십일만칠천삼백구십오
+67890123045607890~육경칠천팔백구십조천이백삼십억사천오백육십만칠천팔백구십
+-2~마이너스 이
+-93~마이너스 구십삼
+-90325~마이너스 구만삼백이십오
+-3234567~마이너스 삼백이십삼만사천오백육십칠

nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py

@@ -15,10 +15,9 @@
 import pytest
 from parameterized import parameterized
 
-from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
 from nemo_text_processing.text_normalization.normalize import Normalizer
 
-from ..utils import CACHE_DIR, parse_test_case_file
+from ..utils import parse_test_case_file
 
 
 class TestCardinal:

tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt

@@ -31,91 +31,6 @@ testTNCardinal() {
   runtest $input
 }
 
-#testTNSpecialText() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_special_text.txt
-#  runtest $input
-#}
-
-#testTNDate() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_date.txt
-#  runtest $input
-#}
-
-#testTNDecimal() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_decimal.txt
-#  runtest $input
-#}
-
-#testTNRange() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_range.txt
-#  runtest $input
-#}
-
-#testTNSerial() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_serial.txt
-#  runtest $input
-#}
-
-#testTNRoman() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_roman.txt
-#  runtest $input
-#}
-
-#testTNElectronic() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_electronic.txt
-#  runtest $input
-#}
-
-#testTNFraction() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_fraction.txt
-#  runtest $input
-#}
-
-#testTNMoney() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_money.txt
-#  runtest $input
-#}
-
-#testTNOrdinal() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_ordinal.txt
-#  runtest $input
-#}
-
-#testTNTelephone() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_telephone.txt
-#  runtest $input
-#}
-
-#testTNTime() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_time.txt
-#  runtest $input
-#}
-
-#testTNMeasure() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_measure.txt
-#  runtest $input
-#}
-
-#testTNWhitelist() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_whitelist.txt
-#  runtest $input
-#}
-
-#testTNWord() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_word.txt
-#  runtest $input
-#}
-
-#testTNAddress() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_address.txt
-#  runtest $input
-#}
-
-#testTNMath() {
-#  input=$TEST_DIR/data_text_normalization/test_cases_math.txt
-#  runtest $input
-#}
-
 # Remove all command-line arguments
 shift $#