Add Vietnamese support using pyvi

Author: kurtisc

Diff for: README.md

@@ -30,6 +30,17 @@ MorphMan supports the following languages:
 - **Japanese**: You must additionally install the _[Japanese Support](https://ankiweb.net/shared/info/3918629684)_ Anki addon
 - **Chinese**: For Anki 2.0, please use [Jieba-Morph](https://github.com/NinKenDo64/Jieba-Morph). Chinese is included in Morphman for Anki 2.1
 - **CJK Characters**: Morphemizer that splits sentence into characters and filters for Chinese-Japanese-Korean logographic/idiographic characters.
+- **Vietnamese**: You must run Anki from source and install [pyvi](https://github.com/trungtv/pyvi) into its virtualenv:
+    - `git clone https://github.com/ankitects/anki.git`
+    - `cd anki`
+    - `make develop`
+        - Make sure you have the dependencies listed in anki/README.development
+    - `source pyenv/bin/activate`
+    - pip install pyvi
+
+    Then run Anki
+    - `./run`
+
 - more languages can be added on request if morpheme-splitting-tools are available for it
 
 See Matt VS Japan's [video tutorial](https://www.youtube.com/watch?v=dVReg8_XnyA)

Diff for: morph/cli.py

@@ -7,7 +7,7 @@
 from collections import Counter
 
 from .morphemes import MorphDb
-from .morphemizer import SpaceMorphemizer, MecabMorphemizer, CjkCharMorphemizer, JiebaMorphemizer
+from .morphemizer import SpaceMorphemizer, MecabMorphemizer, CjkCharMorphemizer, JiebaMorphemizer, VietnameseMorphemizer
 
 
 # hack: typing is compile time anyway, so, nothing bad happens if it fails, the try is to support anki < 2.1.16
@@ -88,6 +88,7 @@ def db_path(db_name):
     'mecab': MecabMorphemizer(),
     'cjkchar': CjkCharMorphemizer(),
     'jieba': JiebaMorphemizer(),
+    'vietnamese': VietnameseMorphemizer(),
 }
 
 

Diff for: morph/morphemizer.py

@@ -5,6 +5,7 @@
 from .deps.zhon.hanzi import characters
 from .mecab_wrapper import getMorphemesMecab, getMecabIdentity
 from .deps.jieba import posseg
+import importlib.util
 
 
 ####################################################################################################
@@ -30,14 +31,23 @@ def getName(self):
         # type: () -> str
         return self.__class__.__name__
 
+    def exists(self):
+        # type: () -> Boolean
+        return True
+
 
 ####################################################################################################
 # Morphemizer Helpers
 ####################################################################################################
 
 def getAllMorphemizers():
     # type: () -> [Morphemizer]
-    return [SpaceMorphemizer(), MecabMorphemizer(), JiebaMorphemizer(), CjkCharMorphemizer()]
+    morphemizers = [SpaceMorphemizer(), MecabMorphemizer(), JiebaMorphemizer(), VietnameseMorphemizer(), CjkCharMorphemizer()]
+    for m in morphemizers:
+        if not m.exists():
+            morphemizers.remove(m)
+
+    return morphemizers
 
 
 def getMorphemizerByName(name):
@@ -90,6 +100,35 @@ def getDescription(self):
         return 'Language w/ Spaces'
 
 
+####################################################################################################
+# Vietnamese Morphemizer
+####################################################################################################
+
+class VietnameseMorphemizer(Morphemizer):
+    """
+    Vietnamese contains many compound words where the polysyllabic morphemes
+    are divided by spaces, so an extra tool - pyvi - is used instead.
+    """
+    def exists(self):
+        """
+        pyvi has large dependencies. To avoid bundling it or forcing users to
+        install it as a dependency, the Vietnamese morphizer only appears if
+        pyvi is importable.
+        """
+        return (importlib.util.find_spec('pyvi') is not None)
+
+    def getMorphemesFromExpr(self, expression):
+        from pyvi import ViTokenizer
+        tokens = SpaceMorphemizer.getMorphemesFromExpr(self, ViTokenizer.tokenize(expression))
+        for word in tokens:
+            word.base = word.base.replace('_', ' ')
+
+        return tokens
+
+    def getDescription(self):
+        return 'Vietnamese'
+
+
 ####################################################################################################
 # CJK Character Morphemizer
 ####################################################################################################

Diff for: test/test_vietnamese_morphemizer.py

@@ -0,0 +1,34 @@
+from morph.morphemizer import getMorphemizerByName
+import unittest
+
+class TestVietnameseMorphemizer(unittest.TestCase):
+    def setUp(self):
+        self.morphemizer = getMorphemizerByName("VietnameseMorphemizer")
+
+    def test_morpheme_generation(self):
+        if self.morphemizer is not None:
+            sentence_1 = ("Trăm năm trong cõi người ta,"
+                          " Chữ tài chữ mệnh khéo là ghét nhau."
+                          " Trải qua một cuộc bể dâu,"
+                          " Những điều trông thấy mà đau đớn lòng.")
+
+            case_1 = ["trăm năm", "trong", "cõi", "người ta", "chữ", "tài", "chữ", "mệnh",
+                      "khéo", "là", "ghét", "nhau", "trải", "qua", "một", "cuộc", "bể dâu",
+                      "những", "điều", "trông", "thấy", "mà", "đau đớn", "lòng"]
+
+            sentence_2 = "Mặt Trời"
+
+            case_2 = ["mặt trời"]
+
+            for idx, m in enumerate(self.morphemizer.getMorphemesFromExpr(sentence_1)):
+                self.assertEqual(m.base, case_1[idx])
+
+            for idx, m in enumerate(self.morphemizer.getMorphemesFromExpr(sentence_2)):
+                self.assertEqual(m.base, case_2[idx])
+
+        else:
+            print('\npyvi is not installed, skipping Vietnamese tests')
+
+
+if __name__ == '__main__':
+    unittest.main()