Version 2.0.5

Author: vthorsteinsson

README.rst

@@ -743,6 +743,9 @@ can be found in the file ``test/toktest_normal_gold_expected.txt``.
 Changelog
 ---------
 
+* Version 2.0.5: Fixed bug where single uppercase letters were erroneously
+  being recognized as abbreviations, causing prepositions such as 'Í' and 'Á'
+  at the beginning of sentences to be misunderstood in ReynirPackage
 * Version 2.0.4: Added imperfect abbreviations (*amk.*, *osfrv.*); recognized
   *klukkan hálf tvö* as a ``TOK.TIME``
 * Version 2.0.3: Fixed bug in ``detokenize()`` where abbreviations, domains

setup.py

@@ -57,7 +57,7 @@ def read(*names, **kwargs):
 
 setup(
     name="tokenizer",
-    version="2.0.4",
+    version="2.0.5",
     license="MIT",
     description="A tokenizer for Icelandic text",
     long_description=u"{0}\n{1}".format(

test/test_tokenizer.py

@@ -1054,11 +1054,14 @@ def test_correct_spaces():
 
 
 def test_abbrev():
-    tokens = list(t.tokenize("Ég las fréttina um IBM t.d. á Mbl."))
+    tokens = list(t.tokenize("Í dag las ég fréttina um IBM t.d. á Mbl."))
     assert tokens == [
         Tok(kind=TOK.S_BEGIN, txt=None, val=(0, None)),
-        Tok(kind=TOK.WORD, txt="Ég", val=None),
+        # We are testing that 'Í' is not an abbreviation
+        Tok(kind=TOK.WORD, txt="Í", val=None),
+        Tok(kind=TOK.WORD, txt="dag", val=None),
         Tok(kind=TOK.WORD, txt="las", val=None),
+        Tok(kind=TOK.WORD, txt="ég", val=None),
         Tok(kind=TOK.WORD, txt="fréttina", val=None),
         Tok(kind=TOK.WORD, txt="um", val=None),
         Tok(