Improved voice assigment. 1) Assign other voices from the same gender if avaiable instead of the same one 2) Check that the regional selection does filter all voices out

Author: jordimas

open_dubbing/main.py

@@ -67,15 +67,17 @@ def log_error_and_exit(msg: str, code: ExitCode):
     exit(code)
 
 
-def check_languages(source_language, target_language, _tts, translation, _stt):
+def check_languages(
+    source_language, target_language, _tts, translation, _stt, target_language_region
+):
     spt = _stt.get_languages()
     translation_languages = translation.get_language_pairs()
     logger().debug(f"check_languages. Pairs {len(translation_languages)}")
 
     tts = _tts.get_languages()
 
     if source_language not in spt:
-        msg = f"source language '{source_language}' is not supported by the speech recognition system. Supported languages: '{spt}"
+        msg = f"source language '{source_language}' is not supported by the speech recognition system. Supported languages: '{spt}'"
         log_error_and_exit(msg, ExitCode.INVALID_LANGUAGE_SPT)
 
     pair = (source_language, target_language)
@@ -84,7 +86,15 @@ def check_languages(source_language, target_language, _tts, translation, _stt):
         log_error_and_exit(msg, ExitCode.INVALID_LANGUAGE_TRANS)
 
     if target_language not in tts:
-        msg = f"target language '{target_language}' is not supported by the text to speech system. Supported languages: '{tts}"
+        msg = f"target language '{target_language}' is not supported by the text to speech system. Supported languages: '{tts}'"
+        log_error_and_exit(msg, ExitCode.INVALID_LANGUAGE_TTS)
+
+    voices = _tts.get_available_voices(language_code=target_language)
+    region_voices = _tts.get_voices_for_region_only(
+        voices=voices, target_language_region=target_language_region
+    )
+    if len(region_voices) == 0:
+        msg = f"filtering by '{target_language_region}' returns no voices for language '{target_language}' in the text to speech system"
         log_error_and_exit(msg, ExitCode.INVALID_LANGUAGE_TTS)
 
 
@@ -275,7 +285,14 @@ def main():
         args.translator, args.nllb_model, args.apertium_server, args.device
     )
 
-    check_languages(source_language, args.target_language, tts, translation, stt)
+    check_languages(
+        source_language,
+        args.target_language,
+        tts,
+        translation,
+        stt,
+        args.target_language_region,
+    )
 
     if not os.path.exists(args.output_directory):
         os.makedirs(args.output_directory)

open_dubbing/text_to_speech.py

@@ -42,18 +42,17 @@ def __init__(self):
     def get_available_voices(self, language_code: str) -> List[Voice]:
         pass
 
-    def get_voices_with_region_preference(
+    def get_voices_for_region_only(
         self, *, voices: List[Voice], target_language_region: str
     ) -> List[Voice]:
         if len(target_language_region) == 0:
             return voices
 
-        voices_copy = voices[:]
+        voices_copy = []
 
         for voice in voices:
             if voice.region.endswith(target_language_region):
-                voices_copy.remove(voice)
-                voices_copy.insert(0, voice)
+                voices_copy.append(voice)
 
         return voices_copy
 
@@ -66,21 +65,39 @@ def assign_voices(
     ) -> Mapping[str, str | None]:
 
         voices = self.get_available_voices(target_language)
-        voices = self.get_voices_with_region_preference(
+        region_voices = self.get_voices_for_region_only(
             voices=voices, target_language_region=target_language_region
         )
 
         voice_assignment = {}
+        used_voices = set()
         for chunk in utterance_metadata:
             speaker_id = chunk["speaker_id"]
             if speaker_id in voice_assignment:
                 continue
 
             gender = chunk["gender"]
-            for voice in voices:
-                if voice.gender.lower() == gender.lower():
+            for voice in region_voices:  # Try to use an unused voice of the same gender
+                if (
+                    voice.name not in used_voices
+                    and voice.gender.lower() == gender.lower()
+                ):
                     voice_assignment[speaker_id] = voice.name
+                    used_voices.add(voice.name)
                     break
+            else:
+                for (
+                    voice
+                ) in region_voices:  # Try to use an already used voice of same gender
+                    if voice.gender.lower() == gender.lower():
+                        voice_assignment[speaker_id] = voice.name
+                        used_voices.add(voice.name)
+                        break
+                else:  # Try to use any other voice of any gender even if it used
+                    for voice in region_voices:
+                        voice_assignment[speaker_id] = voice.name
+                        used_voices.add(voice.name)
+                        break
 
         logger().info(f"text_to_speech.assign_voices. Returns: {voice_assignment}")
         return voice_assignment

tests/text_to_speech_test.py

@@ -353,36 +353,46 @@ def test_get_start_time_of_next_speech_utterance(
             )
             assert result == expected_result
 
-    def test_get_voices_with_region_filter(self):
+    def test_get_voices_for_region_only(self):
         voices = [
             Voice(name="Voice1", gender="Male", region="US"),
             Voice(name="Voice2", gender="Female", region="UK"),
             Voice(name="Voice3", gender="Male", region="IN"),
             Voice(name="Voice4", gender="Female", region="IN"),
         ]
 
-        result = TextToSpeechUT().get_voices_with_region_preference(
+        result = TextToSpeechUT().get_voices_for_region_only(
             voices=voices, target_language_region="UK"
         )
-        assert result[0].region == "UK"
+        assert 1 == len(result)
+        assert "UK" == result[0].region
 
-        result = TextToSpeechUT().get_voices_with_region_preference(
+        result = TextToSpeechUT().get_voices_for_region_only(
             voices=voices, target_language_region="IN"
         )
-        assert result[0].region == "IN"
-        assert result[1].region == "IN"
 
-        result = TextToSpeechUT().get_voices_with_region_preference(
+        assert 2 == len(result)
+        assert "IN" == result[0].region
+        assert "IN" == result[1].region
+
+        result = TextToSpeechUT().get_voices_for_region_only(
             voices=voices, target_language_region=""
         )
-        assert result[0].region == "US"
+        assert 4 == len(result)
+        assert "US" == result[0].region
 
-    def test_assign_voices(self):
+    @pytest.mark.parametrize(
+        "target_language_region, expected_voices",
+        [
+            ("IN", {1: "Voice3"}),
+            ("", {1: "Voice1"}),
+        ],
+    )
+    def test_assign_voices_single_male(self, target_language_region, expected_voices):
         tts = TextToSpeechUT()
 
         utterance_metadata = [
             {
-                "assigned_voice": "en_voice",
                 "speaker_id": 1,
                 "gender": "Male",
             }
@@ -401,9 +411,124 @@ def test_assign_voices(self):
             results = tts.assign_voices(
                 utterance_metadata=utterance_metadata,
                 target_language="",
-                target_language_region="IN",
+                target_language_region=target_language_region,
+            )
+            assert expected_voices == results
+
+    @pytest.mark.parametrize(
+        "target_language_region, expected_voices",
+        [
+            ("IN", {1: "Voice2"}),
+            ("", {1: "Voice1"}),
+        ],
+    )
+    def test_assign_voices_single_male_no_male_voice(
+        self, target_language_region, expected_voices
+    ):
+        tts = TextToSpeechUT()
+
+        utterance_metadata = [
+            {
+                "speaker_id": 1,
+                "gender": "Male",
+            }
+        ]
+
+        voices = [
+            Voice(name="Voice1", gender="Female", region="UK"),
+            Voice(name="Voice2", gender="Female", region="IN"),
+        ]
+
+        tts = TextToSpeechUT()
+
+        with patch.object(tts, "get_available_voices", return_value=voices):
+            results = tts.assign_voices(
+                utterance_metadata=utterance_metadata,
+                target_language="",
+                target_language_region=target_language_region,
+            )
+            assert expected_voices == results
+
+    @pytest.mark.parametrize(
+        "target_language_region, expected_voices",
+        [
+            ("IN", {1: "Voice3", 2: "Voice3"}),
+            ("", {1: "Voice1", 2: "Voice3"}),
+        ],
+    )
+    def test_assign_voices_single_two_males_single_voice(
+        self, target_language_region, expected_voices
+    ):
+        tts = TextToSpeechUT()
+
+        utterance_metadata = [
+            {
+                "speaker_id": 1,
+                "gender": "Male",
+            },
+            {
+                "speaker_id": 2,
+                "gender": "Male",
+            },
+        ]
+
+        voices = [
+            Voice(name="Voice1", gender="Male", region="US"),
+            Voice(name="Voice2", gender="Female", region="US"),
+            Voice(name="Voice3", gender="Male", region="IN"),
+            Voice(name="Voice4", gender="Female", region="IN"),
+        ]
+
+        tts = TextToSpeechUT()
+
+        with patch.object(tts, "get_available_voices", return_value=voices):
+            results = tts.assign_voices(
+                utterance_metadata=utterance_metadata,
+                target_language="",
+                target_language_region=target_language_region,
+            )
+            assert expected_voices == results
+
+    @pytest.mark.parametrize(
+        "target_language_region, expected_voices",
+        [
+            ("IN", {1: "Voice3", 2: "Voice5"}),
+            ("", {1: "Voice1", 2: "Voice3"}),
+        ],
+    )
+    def test_assign_voices_single_two_males_two_voices(
+        self, target_language_region, expected_voices
+    ):
+        tts = TextToSpeechUT()
+
+        utterance_metadata = [
+            {
+                "speaker_id": 1,
+                "gender": "Male",
+            },
+            {
+                "speaker_id": 2,
+                "gender": "Male",
+            },
+        ]
+
+        voices = [
+            Voice(name="Voice1", gender="Male", region="US"),
+            Voice(name="Voice2", gender="Female", region="UK"),
+            Voice(name="Voice3", gender="Male", region="IN"),
+            Voice(name="Voice4", gender="Female", region="IN"),
+            Voice(name="Voice5", gender="Male", region="IN"),
+        ]
+
+        tts = TextToSpeechUT()
+
+        with patch.object(tts, "get_available_voices", return_value=voices):
+            results = tts.assign_voices(
+                utterance_metadata=utterance_metadata,
+                target_language="",
+                target_language_region=target_language_region,
             )
-            assert {1: "Voice3"} == results
+            assert expected_voices == results
 
     def _get_update_utterance_metadata(self):
         return [