Merge pull request #39 from mideind/puncterrors

Puncterrors

Author: vthorsteinsson

src/tokenizer/definitions.py

@@ -54,6 +54,7 @@
 TelnoTuple = Tuple[str, str]
 CurrencyTuple = Tuple[str, Optional[List[str]], Optional[List[str]]]
 
+
 class BIN_Tuple(NamedTuple):
     stofn: str
     utg: int
@@ -62,13 +63,16 @@ class BIN_Tuple(NamedTuple):
     ordmynd: str
     beyging: str
 
+
 BIN_TupleList = Sequence[BIN_Tuple]
 
+
 class PersonNameTuple(NamedTuple):
     name: str
     gender: Optional[str]
     case: Optional[str]
 
+
 PersonNameList = Sequence[PersonNameTuple]
 
 # All possible contents of the Tok.val attribute
@@ -193,6 +197,9 @@ class PersonNameTuple(NamedTuple):
 PUNCT_ENDING_WORD = frozenset(["'", "²", "³"])
 # Punctuation symbols that may occur together
 PUNCT_COMBINATIONS = frozenset(["?", "!", "…"])
+# Punctuation in end of indirect speech that doesn't necessarily end sentences
+PUNCT_INDIRECT_SPEECH = frozenset(["?", "!"])
+
 
 # Single and double quotes
 SQUOTES = "'‚‛‘´"
@@ -204,7 +211,13 @@ class PersonNameTuple(NamedTuple):
 TELNO_PREFIXES = "45678"
 
 # Known telephone country codes
-COUNTRY_CODES = frozenset(("354", "+354", "00354",))
+COUNTRY_CODES = frozenset(
+    (
+        "354",
+        "+354",
+        "00354",
+    )
+)
 
 # Words that can precede a year number; will be assimilated into the year token
 YEAR_WORD = frozenset(("árið", "ársins", "árinu"))
@@ -516,9 +529,9 @@ class PersonNameTuple(NamedTuple):
     "N": "Norður",
 }
 
-_unit_lambda: Callable[[str], str] = lambda unit: unit + r"(?!\w)" if unit[
-    -1
-].isalpha() else unit
+_unit_lambda: Callable[[str], str] = (
+    lambda unit: unit + r"(?!\w)" if unit[-1].isalpha() else unit
+)
 
 SI_UNITS_SET: FrozenSet[str] = frozenset(SI_UNITS.keys())
 SI_UNITS_REGEX_STRING = r"|".join(
@@ -627,7 +640,7 @@ class PersonNameTuple(NamedTuple):
 
 
 def roman_to_int(s: str) -> int:
-    """ Quick and dirty conversion of an already validated Roman numeral to integer """
+    """Quick and dirty conversion of an already validated Roman numeral to integer"""
     # Adapted from http://code.activestate.com/recipes/81611-roman-numerals/
     i = result = 0
     for integer, numeral in ROMAN_NUMERAL_MAP:
@@ -637,6 +650,7 @@ def roman_to_int(s: str) -> int:
     assert i == len(s)
     return result
 
+
 NUMBER_ABBREV = {
     "þús.": 1000,
     "millj.": 10 ** 6,
@@ -1147,7 +1161,7 @@ def roman_to_int(s: str) -> int:
 
 
 def valid_ssn(kt: str) -> bool:
-    """ Validate Icelandic social security number """
+    """Validate Icelandic social security number"""
     if not kt or len(kt) != 11 or kt[6] != "-":
         return False
     m = 11 - sum((ord(kt[i]) - 48) * KT_MAGIC[i] for i in range(9)) % 11

src/tokenizer/tokenizer.py

@@ -1674,20 +1674,21 @@ def parse(self, rt: Tok) -> Iterable[Tok]:
                 # Normalize two periods to one
                 dots, rt = rt.split(2)
                 yield TOK.Punctuation(dots, normalized=".")
-            elif rtxt.startswith(",,") and rtxt[2:3].isalpha():
-                # Probably someone trying to type opening double quotes with commas
-                punct, rt = rt.split(2)
-                yield TOK.Punctuation(punct, normalized="„")
             elif rtxt.startswith(",,"):
-                # Coalesce multiple commas into one normalized comma
-                numcommas = 2
-                for c in rtxt[2:]:
-                    if c == ",":
-                        numcommas += 1
-                    else:
-                        break
-                punct, rt = rt.split(numcommas)
-                yield TOK.Punctuation(punct, normalized=",")
+                if rtxt[2:3].isalnum():
+                    # Probably someone trying to type opening double quotes with commas
+                    punct, rt = rt.split(2)
+                    yield TOK.Punctuation(punct, normalized="„")
+                else:
+                    # Coalesce multiple commas into one normalized comma
+                    numcommas = 2
+                    for c in rtxt[2:]:
+                        if c == ",":
+                            numcommas += 1
+                        else:
+                            break
+                    punct, rt = rt.split(numcommas)
+                    yield TOK.Punctuation(punct, normalized=",")
             elif rtxt[0] in HYPHENS:
                 # Normalize all hyphens the same way
                 punct, rt = rt.split(1)
@@ -2459,6 +2460,23 @@ def parse_sentences(token_stream: Iterator[Tok]) -> Iterator[Tok]:
                     # This token starts a new sentence
                     yield tok_begin_sentence
                     in_sentence = True
+                if (
+                    token.punctuation in PUNCT_INDIRECT_SPEECH
+                    and next_token.punctuation in DQUOTES
+                ):
+                    yield token
+                    token = next_token
+                    next_token = next(token_stream)
+                    if next_token.txt.islower():
+                        # Probably indirect speech
+                        # „Er einhver þarna?“ sagði konan.
+                        yield token
+                        token = next_token
+                        next_token = next(token_stream)
+                    else:
+                        yield token
+                        token = tok_end_sentence
+                        in_sentence = False
                 if token.punctuation in END_OF_SENTENCE and not (
                     token.punctuation
                     == "…"  # Excluding sentences with ellipsis in the middle
@@ -2535,7 +2553,6 @@ def parse_phrases_1(token_stream: Iterator[Tok]) -> Iterator[Tok]:
         # Maintain a one-token lookahead
         token = next(token_stream)
         while True:
-
             next_token = next(token_stream)
             # Coalesce abbreviations and trailing period
             if token.kind == TOK.WORD and next_token.txt == ".":

test/test_tokenizer.py

@@ -58,6 +58,11 @@ def strip_originals(tokens: List[Tok]) -> List[Tok]:
     return tokens
 
 
+def get_text_and_norm(orig: str) -> Tuple[str, str]:
+    toklist = list(t.tokenize(orig))
+    return t.text_from_tokens(toklist), t.normalized_text_from_tokens(toklist)
+
+
 def test_single_tokens() -> None:
 
     TEST_CASES = [
@@ -2319,16 +2324,12 @@ def gen(s: Iterable[str]) -> Iterator[str]:
     g = t.split_into_sentences("Athugum [hvort [setningin sé rétt skilin]].")
     sents = list(g)
     assert len(sents) == 1
-    assert sents == [
-        "Athugum [ hvort [ setningin sé rétt skilin ] ] ."
-    ]
+    assert sents == ["Athugum [ hvort [ setningin sé rétt skilin ] ] ."]
 
     g = t.split_into_sentences("Þessi [ætti [líka að]] vera rétt skilin.")
     sents = list(g)
     assert len(sents) == 1
-    assert sents == [
-        "Þessi [ ætti [ líka að ] ] vera rétt skilin ."
-    ]
+    assert sents == ["Þessi [ ætti [ líka að ] ] vera rétt skilin ."]
 
     # g = t.split_into_sentences("Þessi á [[líka að]] vera rétt skilin.")
     # sents = list(g)
@@ -2340,11 +2341,81 @@ def gen(s: Iterable[str]) -> Iterator[str]:
     # ]
     # Test onesentperline
 
+    # Test whether indirect speech is split up
+    g = t.split_into_sentences("„Er einhver þarna?“ sagði konan.")
+    sents = list(g)
+    assert len(sents) == 1
+    assert sents == ["„ Er einhver þarna ? “ sagði konan ."]
+
+    g = t.split_into_sentences("„Er einhver þarna?“ Maðurinn þorði varla fram.")
+    sents = list(g)
+    assert len(sents) == 2
+    assert sents == ["„ Er einhver þarna ? “", "Maðurinn þorði varla fram ."]
+
+    g = t.split_into_sentences("„Hún hló,“ sagði barnið.")
+    sents = list(g)
+    assert len(sents) == 1
+    assert sents == ["„ Hún hló , “ sagði barnið ."]
+
+    # g = t.split_into_sentences("„Hvað meinarðu??“ sagði barnið.")
+    # sents = list(g)
+    # assert len(sents) == 1
+    # assert sents == ["„ Hvað meinarðu ?? “ sagði barnið ."]
+
 
 def test_normalization() -> None:
-    toklist = list(t.tokenize('Hann sagði: "Þú ert ágæt!".'))
-    assert t.text_from_tokens(toklist) == 'Hann sagði : " Þú ert ágæt ! " .'
-    assert t.normalized_text_from_tokens(toklist) == "Hann sagði : „ Þú ert ágæt ! “ ."
+    text, norm = get_text_and_norm('Hann sagði: "Þú ert ágæt!".')
+
+    assert text == 'Hann sagði : " Þú ert ágæt ! " .'
+    assert norm == "Hann sagði : „ Þú ert ágæt ! “ ."
+
+    text, norm = get_text_and_norm("Hún vinnur í fjármála-og efnahagsráðuneytinu.")
+    assert text == "Hún vinnur í fjármála- og efnahagsráðuneytinu ."
+    assert norm == "Hún vinnur í fjármála- og efnahagsráðuneytinu ."
+
+    text, norm = get_text_and_norm("Þetta er tyrfið...")
+    assert text == "Þetta er tyrfið ..."
+    assert norm == "Þetta er tyrfið …"
+
+    text, norm = get_text_and_norm("Þetta er gaman..")
+    assert text == "Þetta er gaman .."
+    assert norm == "Þetta er gaman ."
+
+    text, norm = get_text_and_norm("Þetta er hvellur.....")
+    assert text == "Þetta er hvellur ....."
+    assert norm == "Þetta er hvellur …"
+
+    text, norm = get_text_and_norm("Þetta er mergjað………")
+    assert text == "Þetta er mergjað ………"
+    assert norm == "Þetta er mergjað …"
+
+    text, norm = get_text_and_norm("Haldið var áfram [...] eftir langt hlé.")
+    assert text == "Haldið var áfram [...] eftir langt hlé ."
+    assert norm == "Haldið var áfram […] eftir langt hlé ."
+
+    text, norm = get_text_and_norm("Þetta er tyrfið,, en við höldum áfram.")
+    assert text == "Þetta er tyrfið ,, en við höldum áfram ."
+    assert norm == "Þetta er tyrfið , en við höldum áfram ."
+
+    text, norm = get_text_and_norm('Hinn svokallaði ,,Galileóhestur" hvarf.')
+    assert text == 'Hinn svokallaði ,, Galileóhestur " hvarf .'
+    assert norm == "Hinn svokallaði „ Galileóhestur “ hvarf ."
+
+    text, norm = get_text_and_norm("Mars - hin rauða pláneta - skín bjart í nótt.")
+    assert text == "Mars - hin rauða pláneta - skín bjart í nótt ."
+    assert norm == "Mars - hin rauða pláneta - skín bjart í nótt ."
+
+    text, norm = get_text_and_norm("Mars – hin rauða pláneta – skín bjart í nótt.")
+    assert text == "Mars – hin rauða pláneta – skín bjart í nótt ."
+    assert norm == "Mars - hin rauða pláneta - skín bjart í nótt ."
+
+    text, norm = get_text_and_norm("Mars — hin rauða pláneta — skín bjart í nótt.")
+    assert text == "Mars — hin rauða pláneta — skín bjart í nótt ."
+    assert norm == "Mars - hin rauða pláneta - skín bjart í nótt ."
+
+    text, norm = get_text_and_norm("Hvernig gastu gert þetta???!!!!!")
+    assert text == "Hvernig gastu gert þetta ???!!!!!"
+    assert norm == "Hvernig gastu gert þetta ?"
 
     toklist = list(t.tokenize('Hann sagði: ,,Þú ert ágæt!!??!".'))
     assert t.text_from_tokens(toklist) == 'Hann sagði : ,, Þú ert ágæt !!??! " .'