Skip to content

Commit d94c723

Browse files
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
1 parent e606249 commit d94c723

File tree

7 files changed

+147
-167
lines changed

7 files changed

+147
-167
lines changed

nemo_text_processing/text_normalization/ko/taggers/cardinal.py

Lines changed: 136 additions & 151 deletions
Large diffs are not rendered by default.

nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,27 +50,21 @@ def __init__(
5050
deterministic: bool = True,
5151
cache_dir: str = None,
5252
overwrite_cache: bool = False,
53-
whitelist: str = None
53+
whitelist: str = None,
5454
):
5555
super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic)
5656

5757
far_file = None
5858
if cache_dir is not None and cache_dir != "None":
5959
os.makedirs(cache_dir, exist_ok=True)
60-
far_file = os.path.join(
61-
cache_dir,
62-
f"ko_tn_{deterministic}_tokenize.far"
63-
)
60+
far_file = os.path.join(cache_dir, f"ko_tn_{deterministic}_tokenize.far")
6461
if not overwrite_cache and far_file and os.path.exists(far_file):
6562
self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
6663
logger.info(f"ClassifyFst.fst was restored from {far_file}.")
6764
else:
6865
cardinal = CardinalFst(deterministic=deterministic)
6966

70-
classify = pynini.union(
71-
72-
pynutil.add_weight(cardinal.fst, 1.1)
73-
)
67+
classify = pynini.union(pynutil.add_weight(cardinal.fst, 1.1))
7468

7569
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
7670
tagger = pynini.closure(token, 1)

nemo_text_processing/text_normalization/ko/verbalizers/verbalize.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ class VerbalizeFst(GraphFst):
3131

3232
def __init__(self, deterministic: bool = True):
3333
super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
34-
34+
3535
cardinal = CardinalFst(deterministic=deterministic)
3636
cardinal_graph = cardinal.fst
3737

38-
self.fst = cardinal_graph.optimize()
38+
self.fst = cardinal_graph.optimize()

nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_
5151
logger.info(f'VerbalizeFinalFst graph was restored from {far_file}.')
5252
else:
5353
verbalize = VerbalizeFst(deterministic=deterministic).fst
54-
#word = WordFst(deterministic=deterministic).fst
54+
# word = WordFst(deterministic=deterministic).fst
5555
types = verbalize
5656

5757
if deterministic:

tests/nemo_text_processing/ko/test_cardinal.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,13 @@
2222

2323

2424
class TestCardinal:
25-
normalizer_ko = Normalizer(lang='ko', cache_dir='export/ko_tn_grammars_lower_cased', overwrite_cache=False, input_case='lower_cased')
25+
normalizer_ko = Normalizer(
26+
lang='ko', cache_dir='export/ko_tn_grammars_lower_cased', overwrite_cache=False, input_case='lower_cased'
27+
)
2628

2729
@parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_cardinal.txt'))
2830
@pytest.mark.run_only_on('CPU')
2931
@pytest.mark.unit
3032
def test_norm(self, test_input, expected):
3133
preds = self.normalizer_ko.normalize(test_input)
3234
assert expected == preds
33-

tests/nemo_text_processing/utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def parse_test_case_file(file_name: str):
3939
Prepares tests pairs for ITN and TN tests
4040
"""
4141
test_pairs = []
42-
with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r', encoding = 'utf-8') as f:
42+
with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r', encoding='utf-8') as f:
4343
for line in f:
4444
components = line.strip("\n").split("~")
4545
spoken = components[0]

tools/text_processing_deployment/pynini_export.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -309,10 +309,10 @@ def parse_args():
309309
)
310310
from nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst
311311
elif args.language == 'rw':
312+
from nemo_text_processing.text_normalization.ko.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst
312313
from nemo_text_processing.text_normalization.rw.taggers.tokenize_and_classify import (
313314
ClassifyFst as TNClassifyFst,
314315
)
315-
from nemo_text_processing.text_normalization.ko.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst
316316
elif args.language == 'ko':
317317
from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import (
318318
ClassifyFst as TNClassifyFst,

0 commit comments

Comments
 (0)