Refactoring of tokenizer.py to shorten very long functions

Author: vthorsteinsson

src/tokenizer/__init__.py

@@ -27,16 +27,36 @@
 """
 
 from .definitions import (
-    TP_LEFT, TP_CENTER, TP_RIGHT, TP_NONE, TP_WORD,
-    EN_DASH, EM_DASH,
-    KLUDGY_ORDINALS_PASS_THROUGH, KLUDGY_ORDINALS_MODIFY, KLUDGY_ORDINALS_TRANSLATE,
-    BIN_Tuple, BIN_TupleList
+    TP_LEFT,
+    TP_CENTER,
+    TP_RIGHT,
+    TP_NONE,
+    TP_WORD,
+    EN_DASH,
+    EM_DASH,
+    KLUDGY_ORDINALS_PASS_THROUGH,
+    KLUDGY_ORDINALS_MODIFY,
+    KLUDGY_ORDINALS_TRANSLATE,
+    BIN_Tuple,
+    BIN_TupleList,
 )
 from .tokenizer import (
-    TOK, Tok, tokenize, tokenize_without_annotation, split_into_sentences,
-    parse_tokens, correct_spaces, detokenize, mark_paragraphs, paragraphs,
-    normalized_text, normalized_text_from_tokens, text_from_tokens,
-    calculate_indexes, generate_rough_tokens, TokenStream
+    TOK,
+    Tok,
+    tokenize,
+    tokenize_without_annotation,
+    split_into_sentences,
+    parse_tokens,
+    correct_spaces,
+    detokenize,
+    mark_paragraphs,
+    paragraphs,
+    normalized_text,
+    normalized_text_from_tokens,
+    text_from_tokens,
+    calculate_indexes,
+    generate_raw_tokens,
+    TokenStream,
 )
 from .abbrev import Abbreviations, ConfigError
 from .version import __version__
@@ -45,14 +65,38 @@
 __copyright__ = "(C) 2021 Miðeind ehf."
 
 __all__ = (
-    "TP_LEFT","TP_CENTER", "TP_RIGHT", "TP_NONE", "TP_WORD",
-    "EN_DASH", "EM_DASH",
-    "KLUDGY_ORDINALS_PASS_THROUGH", "KLUDGY_ORDINALS_MODIFY", "KLUDGY_ORDINALS_TRANSLATE",
-    "BIN_Tuple", "BIN_TupleList",
-    "TOK", "Tok", "tokenize", "tokenize_without_annotation", "split_into_sentences",
-    "parse_tokens", "correct_spaces", "detokenize", "mark_paragraphs", "paragraphs",
-    "normalized_text", "normalized_text_from_tokens", "text_from_tokens",
-    "calculate_indexes", "generate_rough_tokens", "TokenStream",
-    "Abbreviations", "ConfigError", "__version__", "__author__", "__copyright__"
+    "__author__",
+    "__copyright__",
+    "__version__",
+    "Abbreviations",
+    "BIN_Tuple",
+    "BIN_TupleList",
+    "calculate_indexes",
+    "ConfigError",
+    "correct_spaces",
+    "detokenize",
+    "EM_DASH",
+    "EN_DASH",
+    "generate_raw_tokens",
+    "KLUDGY_ORDINALS_MODIFY",
+    "KLUDGY_ORDINALS_PASS_THROUGH",
+    "KLUDGY_ORDINALS_TRANSLATE",
+    "mark_paragraphs",
+    "normalized_text_from_tokens",
+    "normalized_text",
+    "paragraphs",
+    "parse_tokens",
+    "split_into_sentences",
+    "text_from_tokens",
+    "Tok",
+    "TOK",
+    "tokenize_without_annotation",
+    "tokenize",
+    "TokenStream",
+    "TP_CENTER",
+    "TP_LEFT",
+    "TP_NONE",
+    "TP_RIGHT",
+    "TP_WORD",
 )