Skip to content

Commit 3e4ac3e

Browse files
pre-commit-ci[bot]pre-commit-ci[bot]
authored
[pre-commit.ci] pre-commit suggestions (#270)
* [pre-commit.ci] pre-commit suggestions updates: - [github.com/pre-commit/pre-commit-hooks: v4.3.0 → v5.0.0](pre-commit/pre-commit-hooks@v4.3.0...v5.0.0) - [github.com/PyCQA/flake8: 7.1.1 → 7.2.0](PyCQA/flake8@7.1.1...7.2.0) - [github.com/PyCQA/isort: 5.12.0 → 6.0.1](PyCQA/isort@5.12.0...6.0.1) - [github.com/psf/black: 19.10b0 → 25.1.0](psf/black@19.10b0...25.1.0) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 171de10 commit 3e4ac3e

File tree

192 files changed

+927
-454
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

192 files changed

+927
-454
lines changed

‎.pre-commit-config.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,30 +22,30 @@ ci:
2222

2323
repos:
2424
- repo: https://github.com/pre-commit/pre-commit-hooks
25-
rev: v4.3.0
25+
rev: v5.0.0
2626
hooks:
2727
- id: check-yaml
2828
- id: check-case-conflict
2929
- id: detect-private-key
3030
- id: requirements-txt-fixer
3131

3232
- repo: https://github.com/PyCQA/flake8
33-
rev: 7.1.1
33+
rev: 7.2.0
3434
hooks:
3535
- id: flake8
3636
args:
3737
- --select=W605
3838

3939
- repo: https://github.com/PyCQA/isort
40-
rev: 5.12.0
40+
rev: 6.0.1
4141
hooks:
4242
- id: isort
4343
name: Format imports
4444
args: [ --multi-line=3, --trailing-comma, --force-grid-wrap=0, --use-parentheses, --line-width=119, -rc, -ws ]
4545
exclude: docs/
4646

4747
- repo: https://github.com/psf/black
48-
rev: 19.10b0
48+
rev: 25.1.0
4949
hooks:
5050
- id: black
5151
name: Format code

‎nemo_text_processing/fst_alignment/alignment.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ def parse_args():
9696

9797

9898
EPS = "<eps>"
99-
WHITE_SPACE = "\u23B5"
99+
WHITE_SPACE = "\u23b5"
100100
ITN_MODE = "itn"
101101
TN_MODE = "tn"
102102
tn_item_special_chars = ["$", "\\", ":", "+", "-", "="]

‎nemo_text_processing/hybrid/utils.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ def remove_punctuation(text: str, remove_spaces=True, do_lower=True, lang="en",
482482

483483
text = re.sub(r" +", " ", text)
484484
if remove_spaces:
485-
text = text.replace(" ", "").replace("\u00A0", "").strip()
485+
text = text.replace(" ", "").replace("\u00a0", "").strip()
486486

487487
if do_lower:
488488
text = text.lower()
@@ -515,7 +515,11 @@ def _relax_diff(text):
515515
return acceptable
516516

517517

518-
def get_labels(targets: List[str], norm_texts_weights: List[Tuple[str, str]], lang="en",) -> List[List[str]]:
518+
def get_labels(
519+
targets: List[str],
520+
norm_texts_weights: List[Tuple[str, str]],
521+
lang="en",
522+
) -> List[List[str]]:
519523
"""
520524
Assign labels to generated normalization options (1 - for ground truth, 0 - other options)
521525
Args:
@@ -605,7 +609,14 @@ def print_df(df):
605609
prints data frame
606610
"""
607611
with pd.option_context(
608-
"display.max_rows", None, "display.max_columns", None, "display.width", 1000, "display.max_colwidth", 400,
612+
"display.max_rows",
613+
None,
614+
"display.max_columns",
615+
None,
616+
"display.width",
617+
1000,
618+
"display.max_colwidth",
619+
400,
609620
):
610621
print(df)
611622

‎nemo_text_processing/inverse_text_normalization/ar/taggers/cardinal.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ def __init__(self, tn_cardinal):
3333
self.graph = pynini.invert(tn_cardinal.cardinal_numbers).optimize()
3434

3535
optional_minus_graph = pynini.closure(
36-
pynutil.insert("negative: ") + pynini.cross("سالب", '"-"') + NEMO_SPACE, 0, 1,
36+
pynutil.insert("negative: ") + pynini.cross("سالب", '"-"') + NEMO_SPACE,
37+
0,
38+
1,
3739
)
3840

3941
final_graph = optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')

‎nemo_text_processing/inverse_text_normalization/ar/taggers/decimal.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@ def __init__(self, tn_decimal):
3636
super().__init__(name="decimal", kind="classify")
3737

3838
optional_graph_negative = pynini.closure(
39-
pynutil.insert("negative: ") + pynini.cross("سالب", '"true"') + delete_extra_space, 0, 1,
39+
pynutil.insert("negative: ") + pynini.cross("سالب", '"true"') + delete_extra_space,
40+
0,
41+
1,
4042
)
4143

4244
graph_fractional_part = pynini.invert(tn_decimal.graph_fractional).optimize()

‎nemo_text_processing/inverse_text_normalization/ar/verbalizers/word.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,6 @@ def __init__(self):
2727
super().__init__(name="word", kind="verbalize")
2828
chars = pynini.closure(NEMO_CHAR - " ", 1)
2929
char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"")
30-
graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA)
30+
graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA)
3131

3232
self.fst = graph.optimize()

‎nemo_text_processing/inverse_text_normalization/en/taggers/cardinal.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,10 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED):
207207
+ graph_in_thousands
208208
)
209209

210-
graph = pynini.union((graph_int | graph_ind) + delete_space + graph_hundreds, graph_zero,)
210+
graph = pynini.union(
211+
(graph_int | graph_ind) + delete_space + graph_hundreds,
212+
graph_zero,
213+
)
211214

212215
graph = graph @ pynini.union(
213216
pynutil.delete(pynini.closure("0")) + pynini.difference(NEMO_DIGIT, "0") + pynini.closure(NEMO_DIGIT), "0"

‎nemo_text_processing/inverse_text_normalization/en/taggers/date.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,11 @@ def __init__(self, ordinal: GraphFst, input_case: str):
165165
+ pynutil.add_weight(year_graph, -YEAR_WEIGHT)
166166
+ pynutil.insert("\"")
167167
)
168-
optional_graph_year = pynini.closure(graph_year, 0, 1,)
168+
optional_graph_year = pynini.closure(
169+
graph_year,
170+
0,
171+
1,
172+
)
169173
graph_mdy = month_graph + (
170174
(delete_extra_space + day_graph) | graph_year | (delete_extra_space + day_graph + graph_year)
171175
)

‎nemo_text_processing/inverse_text_normalization/en/taggers/decimal.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,9 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED):
9797
point = pynutil.delete("point")
9898

9999
optional_graph_negative = pynini.closure(
100-
pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space, 0, 1,
100+
pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space,
101+
0,
102+
1,
101103
)
102104

103105
graph_fractional = pynutil.insert("fractional_part: \"") + graph_decimal + pynutil.insert("\"")

‎nemo_text_processing/inverse_text_normalization/en/taggers/electronic.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,13 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED):
106106
delete_extra_space
107107
+ url_symbols
108108
+ delete_extra_space
109-
+ (domain | pynini.closure(accepted_username + delete_extra_space,) + accepted_username)
109+
+ (
110+
domain
111+
| pynini.closure(
112+
accepted_username + delete_extra_space,
113+
)
114+
+ accepted_username
115+
)
110116
)
111117

112118
protocol_default = (

‎nemo_text_processing/inverse_text_normalization/en/taggers/measure.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPU
5858
graph_unit_plural = pynini.compose(casing_graph, graph_unit_plural).optimize()
5959

6060
optional_graph_negative = pynini.closure(
61-
pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space, 0, 1,
61+
pynutil.insert("negative: ") + pynini.cross(MINUS, "\"true\"") + delete_extra_space,
62+
0,
63+
1,
6264
)
6365

6466
unit_singular = convert_space(graph_unit_singular)

‎nemo_text_processing/inverse_text_normalization/en/taggers/money.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, input_case: str = INPU
5353
# "one fifty" -> "one hundred fifty"
5454
with_hundred = pynini.compose(
5555
pynini.closure(NEMO_NOT_SPACE) + pynini.accep(" ") + pynutil.insert("hundred ") + NEMO_SIGMA,
56-
pynini.compose(cardinal_graph, NEMO_DIGIT ** 3),
56+
pynini.compose(cardinal_graph, NEMO_DIGIT**3),
5757
)
5858
cardinal_graph |= with_hundred
5959
graph_decimal_final = decimal.final_graph_wo_negative

‎nemo_text_processing/inverse_text_normalization/en/taggers/telephone.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def get_serial_number(cardinal):
4040
"""
4141

4242
digit = pynini.compose(cardinal.graph_no_exception, NEMO_DIGIT)
43-
two_digit = pynutil.add_weight(pynini.compose(cardinal.graph_two_digit, NEMO_DIGIT ** 2), 0.002)
43+
two_digit = pynutil.add_weight(pynini.compose(cardinal.graph_two_digit, NEMO_DIGIT**2), 0.002)
4444
character = digit | two_digit | NEMO_ALPHA
4545
sequence = (NEMO_LOWER_NOT_A | digit) + pynini.closure(pynutil.delete(" ") + character, 2)
4646
sequence |= character + pynini.closure(pynutil.delete(" ") + (digit | NEMO_ALPHA), 2)
@@ -116,7 +116,7 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED):
116116
triple_digit.invert()
117117

118118
# to handle cases like "one twenty three"
119-
two_digit_cardinal = pynini.compose(cardinal.graph_no_exception, NEMO_DIGIT ** 2)
119+
two_digit_cardinal = pynini.compose(cardinal.graph_no_exception, NEMO_DIGIT**2)
120120
double_digit_to_digit = (
121121
pynini.compose(double_digit, str_to_digit + pynutil.delete(" ") + str_to_digit) | two_digit_cardinal
122122
)
@@ -139,7 +139,7 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED):
139139

140140
number_part = pynini.compose(
141141
single_double_or_triple_digit,
142-
NEMO_DIGIT ** 3 + pynutil.insert("-") + NEMO_DIGIT ** 3 + pynutil.insert("-") + NEMO_DIGIT ** 4,
142+
NEMO_DIGIT**3 + pynutil.insert("-") + NEMO_DIGIT**3 + pynutil.insert("-") + NEMO_DIGIT**4,
143143
).optimize()
144144
number_part = pynutil.insert("number_part: \"") + number_part.optimize() + pynutil.insert("\"")
145145

@@ -156,24 +156,24 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED):
156156
graph = optional_country_code + number_part
157157

158158
# credit card number
159-
space_four_digits = insert_space + NEMO_DIGIT ** 4
159+
space_four_digits = insert_space + NEMO_DIGIT**4
160160
space_five_digits = space_four_digits + NEMO_DIGIT
161161
space_six_digits = space_five_digits + NEMO_DIGIT
162162
credit_card_graph = pynini.compose(
163163
single_double_or_triple_digit,
164-
NEMO_DIGIT ** 4 + (space_six_digits | (space_four_digits ** 2)) + space_four_digits,
164+
NEMO_DIGIT**4 + (space_six_digits | (space_four_digits**2)) + space_four_digits,
165165
).optimize()
166166

167167
credit_card_graph |= pynini.compose(
168-
single_double_or_triple_digit, NEMO_DIGIT ** 4 + space_six_digits + space_five_digits
168+
single_double_or_triple_digit, NEMO_DIGIT**4 + space_six_digits + space_five_digits
169169
).optimize()
170170

171171
graph |= pynutil.insert("number_part: \"") + credit_card_graph.optimize() + pynutil.insert("\"")
172172

173173
# SSN
174174
ssn_graph = pynini.compose(
175175
single_double_or_triple_digit,
176-
NEMO_DIGIT ** 3 + pynutil.insert("-") + NEMO_DIGIT ** 2 + pynutil.insert("-") + NEMO_DIGIT ** 4,
176+
NEMO_DIGIT**3 + pynutil.insert("-") + NEMO_DIGIT**2 + pynutil.insert("-") + NEMO_DIGIT**4,
177177
).optimize()
178178
graph |= pynutil.insert("number_part: \"") + ssn_graph.optimize() + pynutil.insert("\"")
179179

‎nemo_text_processing/inverse_text_normalization/en/taggers/time.py

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,32 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED):
7171
graph_minute_double = pynini.union(*labels_minute_double) @ cardinal
7272

7373
graph_minute_verbose = pynini.cross("half", "30") | pynini.cross("quarter", "15")
74-
oclock = pynini.cross(pynini.union("o' clock", "o clock", "o'clock", "oclock", "hundred hours",), "",)
74+
oclock = pynini.cross(
75+
pynini.union(
76+
"o' clock",
77+
"o clock",
78+
"o'clock",
79+
"oclock",
80+
"hundred hours",
81+
),
82+
"",
83+
)
7584

7685
if input_case == INPUT_CASED:
7786
minute_to_graph = capitalized_input_graph(minute_to_graph)
7887
graph_minute_single = capitalized_input_graph(graph_minute_single)
7988
graph_minute_double = capitalized_input_graph(graph_minute_double)
8089
graph_minute_verbose |= pynini.cross("Half", "30") | pynini.cross("Quarter", "15")
81-
oclock |= pynini.cross(pynini.union("O' clock", "O clock", "O'clock", "Oclock", "Hundred hours",), "",)
90+
oclock |= pynini.cross(
91+
pynini.union(
92+
"O' clock",
93+
"O clock",
94+
"O'clock",
95+
"Oclock",
96+
"Hundred hours",
97+
),
98+
"",
99+
)
82100

83101
final_graph_hour = pynutil.insert("hours: \"") + graph_hour + pynutil.insert("\"")
84102
graph_minute = (

‎nemo_text_processing/inverse_text_normalization/en/verbalizers/whitelist.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,5 @@ def __init__(self):
3535
+ pynini.closure(NEMO_CHAR - " ", 1)
3636
+ pynutil.delete("\"")
3737
)
38-
graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA)
38+
graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA)
3939
self.fst = graph.optimize()

‎nemo_text_processing/inverse_text_normalization/en/verbalizers/word.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,6 @@ def __init__(self):
2929
super().__init__(name="word", kind="verbalize")
3030
chars = pynini.closure(NEMO_CHAR - " ", 1)
3131
char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"")
32-
graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA)
32+
graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA)
3333

3434
self.fst = graph.optimize()

‎nemo_text_processing/inverse_text_normalization/es/taggers/cardinal.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -160,18 +160,13 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED):
160160
self.graph_no_exception = graph.optimize()
161161

162162
# save self.numbers_up_to_thousand for use in DecimalFst
163-
digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT ** 2) | (NEMO_DIGIT ** 3)
163+
digits_up_to_thousand = NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3)
164164
numbers_up_to_thousand = pynini.compose(self.graph_no_exception, digits_up_to_thousand).optimize()
165165
self.numbers_up_to_thousand = numbers_up_to_thousand.optimize()
166166

167167
# save self.numbers_up_to_million for use in DecimalFst
168168
digits_up_to_million = (
169-
NEMO_DIGIT
170-
| (NEMO_DIGIT ** 2)
171-
| (NEMO_DIGIT ** 3)
172-
| (NEMO_DIGIT ** 4)
173-
| (NEMO_DIGIT ** 5)
174-
| (NEMO_DIGIT ** 6)
169+
NEMO_DIGIT | (NEMO_DIGIT**2) | (NEMO_DIGIT**3) | (NEMO_DIGIT**4) | (NEMO_DIGIT**5) | (NEMO_DIGIT**6)
175170
)
176171
numbers_up_to_million = pynini.compose(graph, digits_up_to_million).optimize()
177172
self.numbers_up_to_million = numbers_up_to_million.optimize()

‎nemo_text_processing/inverse_text_normalization/es/taggers/electronic.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,13 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED):
136136
delete_extra_space
137137
+ symbols
138138
+ delete_extra_space
139-
+ (domain | pynini.closure(accepted_username + delete_extra_space,) + accepted_username)
139+
+ (
140+
domain
141+
| pynini.closure(
142+
accepted_username + delete_extra_space,
143+
)
144+
+ accepted_username
145+
)
140146
)
141147

142148
protocol_default = (

‎nemo_text_processing/inverse_text_normalization/es/taggers/ordinal.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,13 @@ def __init__(self, cardinal: GraphFst, input_case: str = INPUT_LOWER_CASED):
6262

6363
full_graph_ties = graph_ties | (graph_ties + pynini.cross(" ", "y") + graph_digit)
6464

65-
ordinal_graph_union = pynini.union(graph_digit, graph_teens, graph_twenties, full_graph_ties, graph_hundreds,)
65+
ordinal_graph_union = pynini.union(
66+
graph_digit,
67+
graph_teens,
68+
graph_twenties,
69+
full_graph_ties,
70+
graph_hundreds,
71+
)
6672

6773
accept_o_endings = NEMO_SIGMA + pynini.accep("o")
6874
accept_a_endings = NEMO_SIGMA + pynini.accep("a")

‎nemo_text_processing/inverse_text_normalization/es/taggers/telephone.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,10 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED):
110110
# Denormalized phone numbers are grouped in sets of 3 or 4 digits
111111
group_of_two = pynini.union(doubled_digit, digit_twice, double_digits)
112112

113-
group_of_three = pynini.union(tripled_digit, single_digits + pynutil.delete(" ") + group_of_two,)
113+
group_of_three = pynini.union(
114+
tripled_digit,
115+
single_digits + pynutil.delete(" ") + group_of_two,
116+
)
114117

115118
group_of_four = pynini.union(
116119
group_of_two + pynutil.delete(" ") + group_of_two,

‎nemo_text_processing/inverse_text_normalization/es/verbalizers/whitelist.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,5 @@ def __init__(self):
3434
+ pynini.closure(NEMO_CHAR - " ", 1)
3535
+ pynutil.delete("\"")
3636
)
37-
graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA)
37+
graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA)
3838
self.fst = graph.optimize()

‎nemo_text_processing/inverse_text_normalization/es/verbalizers/word.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,6 @@ def __init__(self):
2828
super().__init__(name="word", kind="verbalize")
2929
chars = pynini.closure(NEMO_CHAR - " ", 1)
3030
char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"")
31-
graph = char @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA)
31+
graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA)
3232

3333
self.fst = graph.optimize()

‎nemo_text_processing/inverse_text_normalization/fr/graph_utils.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@
3535
NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize()
3636
NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize()
3737
NEMO_HEX = pynini.union(*string.hexdigits).optimize()
38-
NEMO_NON_BREAKING_SPACE = u"\u00A0"
38+
NEMO_NON_BREAKING_SPACE = u"\u00a0"
3939
NEMO_SPACE = " "
40-
NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00A0").optimize()
40+
NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize()
4141
NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize()
4242
NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize()
4343

@@ -188,4 +188,4 @@ def delete_tokens(self, fst) -> 'pynini.FstLike':
188188
+ delete_space
189189
+ pynutil.delete("}")
190190
)
191-
return res @ pynini.cdrewrite(pynini.cross(u"\u00A0", " "), "", "", NEMO_SIGMA)
191+
return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA)

0 commit comments

Comments
 (0)