feat(vc): allow multiple target audio files

Author: eginhard

TTS/api.py

@@ -4,7 +4,7 @@
 import tempfile
 import warnings
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
 
 from torch import nn
 
@@ -388,7 +388,7 @@ def tts_to_file(
     def voice_conversion(
         self,
         source_wav: str,
-        target_wav: str,
+        target_wav: Union[str, list[str]],
     ):
         """Voice conversion with FreeVC. Convert source wav to target speaker.
 
@@ -406,7 +406,7 @@ def voice_conversion(
     def voice_conversion_to_file(
         self,
         source_wav: str,
-        target_wav: str,
+        target_wav: Union[str, list[str]],
         file_path: str = "output.wav",
         pipe_out=None,
     ) -> str:
@@ -429,9 +429,10 @@ def voice_conversion_to_file(
     def tts_with_vc(
         self,
         text: str,
-        language: str = None,
-        speaker_wav: str = None,
-        speaker: str = None,
+        *,
+        language: Optional[str] = None,
+        speaker_wav: Union[str, list[str]],
+        speaker: Optional[str] = None,
         split_sentences: bool = True,
     ):
         """Convert text to speech with voice conversion.
@@ -471,10 +472,11 @@ def tts_with_vc(
     def tts_with_vc_to_file(
         self,
         text: str,
-        language: str = None,
-        speaker_wav: str = None,
+        *,
+        language: Optional[str] = None,
+        speaker_wav: Union[str, list[str]],
         file_path: str = "output.wav",
-        speaker: str = None,
+        speaker: Optional[str] = None,
         split_sentences: bool = True,
         pipe_out=None,
     ) -> str:

TTS/bin/synthesize.py

@@ -275,13 +275,14 @@ def parse_args(arg_list: Optional[list[str]]) -> argparse.Namespace:
         "--source_wav",
         type=str,
         default=None,
-        help="Original audio file to convert in the voice of the target_wav",
+        help="Original audio file to convert into the voice of the target_wav",
     )
     parser.add_argument(
         "--target_wav",
         type=str,
+        nargs="*",
         default=None,
-        help="Target audio file to convert in the voice of the source_wav",
+        help="Audio file(s) of the target voice into which to convert the source_wav",
     )
 
     parser.add_argument(

TTS/utils/synthesizer.py

@@ -274,8 +274,11 @@ def save_wav(self, wav: List[int], path: str, pipe_out=None) -> None:
             wav = np.array(wav)
         save_wav(wav=wav, path=path, sample_rate=self.output_sample_rate, pipe_out=pipe_out)
 
-    def voice_conversion(self, source_wav: str, target_wav: str, **kwargs) -> List[int]:
+    def voice_conversion(self, source_wav: str, target_wav: Union[str, list[str]], **kwargs) -> List[int]:
         start_time = time.time()
+
+        if not isinstance(target_wav, list):
+            target_wav = [target_wav]
         output = self.vc_model.voice_conversion(source_wav, target_wav, **kwargs)
         if self.vocoder_model is not None:
             output = self.vocoder_model.inference(output)

TTS/vc/layers/freevc/speaker_encoder/speaker_encoder.py

@@ -115,7 +115,7 @@ def compute_partial_slices(n_samples: int, rate, min_coverage):
 
         return wav_slices, mel_slices
 
-    def embed_utterance(self, wav: np.ndarray, return_partials=False, rate=1.3, min_coverage=0.75):
+    def embed_utterance(self, wav: np.ndarray, return_partials=False, rate=1.3, min_coverage=0.75) -> torch.Tensor:
         """
         Computes an embedding for a single utterance. The utterance is divided in partial
         utterances and an embedding is computed for each. The complete utterance embedding is the

TTS/vc/models/freevc.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Union
 
 import librosa
 import numpy as np
@@ -386,7 +386,7 @@ def forward(
         return o, ids_slice, spec_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
 
     @torch.inference_mode()
-    def inference(self, c, g=None, mel=None, c_lengths=None):
+    def inference(self, c, g=None, c_lengths=None):
         """
         Inference pass of the model
 
@@ -401,9 +401,6 @@ def inference(self, c, g=None, mel=None, c_lengths=None):
         """
         if c_lengths is None:
             c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device)
-        if not self.use_spk:
-            g = self.enc_spk.embed_utterance(mel)
-            g = g.unsqueeze(-1)
         z_p, m_p, logs_p, c_mask = self.enc_p(c, c_lengths)
         z = self.flow(z_p, c_mask, g=g, reverse=True)
         o = self.dec(z * c_mask, g=g)
@@ -434,45 +431,47 @@ def load_audio(self, wav):
         return wav.float()
 
     @torch.inference_mode()
-    def voice_conversion(self, src, tgt):
+    def voice_conversion(self, src: Union[str, torch.Tensor], tgt: list[Union[str, torch.Tensor]]):
         """
         Voice conversion pass of the model.
 
         Args:
             src (str or torch.Tensor): Source utterance.
-            tgt (str or torch.Tensor): Target utterance.
+            tgt (list of str or torch.Tensor): Target utterances.
 
         Returns:
             torch.Tensor: Output tensor.
         """
 
-        wav_tgt = self.load_audio(tgt).cpu().numpy()
-        wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20)
-
-        if self.config.model_args.use_spk:
-            g_tgt = self.enc_spk_ex.embed_utterance(wav_tgt)[None, :, None]
-        else:
-            wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).to(self.device)
-            mel_tgt = mel_spectrogram_torch(
-                wav_tgt,
-                self.config.audio.filter_length,
-                self.config.audio.n_mel_channels,
-                self.config.audio.input_sample_rate,
-                self.config.audio.hop_length,
-                self.config.audio.win_length,
-                self.config.audio.mel_fmin,
-                self.config.audio.mel_fmax,
-            )
         # src
         wav_src = self.load_audio(src)
         c = self.extract_wavlm_features(wav_src[None, :])
 
-        if self.config.model_args.use_spk:
-            audio = self.inference(c, g=g_tgt)
-        else:
-            audio = self.inference(c, mel=mel_tgt.transpose(1, 2))
-        audio = audio[0][0].data.cpu().float().numpy()
-        return audio
+        # tgt
+        g_tgts = []
+        for tg in tgt:
+            wav_tgt = self.load_audio(tg).cpu().numpy()
+            wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20)
+
+            if self.config.model_args.use_spk:
+                g_tgts.append(self.enc_spk_ex.embed_utterance(wav_tgt)[None, :, None])
+            else:
+                wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).to(self.device)
+                mel_tgt = mel_spectrogram_torch(
+                    wav_tgt,
+                    self.config.audio.filter_length,
+                    self.config.audio.n_mel_channels,
+                    self.config.audio.input_sample_rate,
+                    self.config.audio.hop_length,
+                    self.config.audio.win_length,
+                    self.config.audio.mel_fmin,
+                    self.config.audio.mel_fmax,
+                )
+                g_tgts.append(self.enc_spk.embed_utterance(mel_tgt.transpose(1, 2)).unsqueeze(-1))
+
+        g_tgt = torch.stack(g_tgts).mean(dim=0)
+        audio = self.inference(c, g=g_tgt)
+        return audio[0][0].data.cpu().float().numpy()
 
     def eval_step(): ...
 

TTS/vc/models/knnvc.py

@@ -172,7 +172,7 @@ def inference(self) -> None: ...
     def voice_conversion(
         self,
         source: PathOrTensor,
-        target: Union[PathOrTensor, list[PathOrTensor]],
+        target: list[PathOrTensor],
         topk: Optional[int] = None,
     ) -> torch.Tensor:
         if not isinstance(target, list):

TTS/vc/models/openvoice.py

@@ -296,19 +296,25 @@ def extract_se(self, audio: Union[str, torch.Tensor]) -> tuple[torch.Tensor, tor
         return g, spec
 
     @torch.inference_mode()
-    def voice_conversion(self, src: Union[str, torch.Tensor], tgt: Union[str, torch.Tensor]) -> npt.NDArray[np.float32]:
+    def voice_conversion(
+        self, src: Union[str, torch.Tensor], tgt: list[Union[str, torch.Tensor]]
+    ) -> npt.NDArray[np.float32]:
         """
         Voice conversion pass of the model.
 
         Args:
             src (str or torch.Tensor): Source utterance.
-            tgt (str or torch.Tensor): Target utterance.
+            tgt (list of str or torch.Tensor): Target utterance.
 
         Returns:
             Output numpy array.
         """
         src_se, src_spec = self.extract_se(src)
-        tgt_se, _ = self.extract_se(tgt)
+        tgt_ses = []
+        for tg in tgt:
+            tgt_se, _ = self.extract_se(tg)
+            tgt_ses.append(tgt_se)
+        tgt_se = torch.stack(tgt_ses).mean(dim=0)
 
         aux_input = {"g_src": src_se, "g_tgt": tgt_se}
         audio = self.inference(src_spec, aux_input)

tests/zoo_tests/test_models.py

@@ -71,8 +71,9 @@ def test_models(tmp_path, model_name, manager):
         run_main(main, [*args, "--text", "This is an example.", *speaker_arg, *language_arg])
     elif "voice_conversion_models" in model_name:
         speaker_wav = os.path.join(get_tests_data_path(), "ljspeech", "wavs", "LJ001-0001.wav")
-        reference_wav = os.path.join(get_tests_data_path(), "ljspeech", "wavs", "LJ001-0032.wav")
-        run_main(main, [*args, "--source_wav", speaker_wav, "--target_wav", reference_wav])
+        reference_wav1 = os.path.join(get_tests_data_path(), "ljspeech", "wavs", "LJ001-0028.wav")
+        reference_wav2 = os.path.join(get_tests_data_path(), "ljspeech", "wavs", "LJ001-0032.wav")
+        run_main(main, [*args, "--source_wav", speaker_wav, "--target_wav", reference_wav1, reference_wav2])
     else:
         # only download the model
         manager.download_model(model_name)