From fa5cd2b5bfda3b2022f43fae430e37c8c7919fea Mon Sep 17 00:00:00 2001 From: Moshe Gordon Radian Date: Fri, 8 Mar 2024 00:35:06 +0200 Subject: [PATCH] feat: separate voices from wasm package to hopefully be able to fix the github pages issue, either via regular build or uploading a compiled app --- .../cpp => public}/voices/.gitattributes | 0 .../oddvoices/cpp => public}/voices/air.voice | 0 .../cpp => public}/voices/cicada.voice | 0 .../cpp => public}/voices/quake.voice | 0 musicxml-singer-with-oddvoices/src/App.tsx | 159 +++++++----------- .../src/MediaControls.tsx | 57 +++++++ .../src/UploadButton.tsx | 88 ++++++++++ .../src/oddvoices/cpp/Makefile | 2 +- .../src/oddvoices/cpp/oddvoices_wasm.cpp | 11 +- .../src/oddvoices/index.tsx | 64 ++++++- .../src/oddvoices/oddvoicesUtils.tsx | 8 +- 11 files changed, 275 insertions(+), 114 deletions(-) rename musicxml-singer-with-oddvoices/{src/oddvoices/cpp => public}/voices/.gitattributes (100%) rename musicxml-singer-with-oddvoices/{src/oddvoices/cpp => public}/voices/air.voice (100%) rename musicxml-singer-with-oddvoices/{src/oddvoices/cpp => public}/voices/cicada.voice (100%) rename musicxml-singer-with-oddvoices/{src/oddvoices/cpp => public}/voices/quake.voice (100%) create mode 100644 musicxml-singer-with-oddvoices/src/MediaControls.tsx create mode 100644 musicxml-singer-with-oddvoices/src/UploadButton.tsx diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/.gitattributes b/musicxml-singer-with-oddvoices/public/voices/.gitattributes similarity index 100% rename from musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/.gitattributes rename to musicxml-singer-with-oddvoices/public/voices/.gitattributes diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/air.voice b/musicxml-singer-with-oddvoices/public/voices/air.voice similarity index 100% rename from musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/air.voice rename to musicxml-singer-with-oddvoices/public/voices/air.voice diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/cicada.voice b/musicxml-singer-with-oddvoices/public/voices/cicada.voice similarity index 100% rename from musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/cicada.voice rename to musicxml-singer-with-oddvoices/public/voices/cicada.voice diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/quake.voice b/musicxml-singer-with-oddvoices/public/voices/quake.voice similarity index 100% rename from musicxml-singer-with-oddvoices/src/oddvoices/cpp/voices/quake.voice rename to musicxml-singer-with-oddvoices/public/voices/quake.voice diff --git a/musicxml-singer-with-oddvoices/src/App.tsx b/musicxml-singer-with-oddvoices/src/App.tsx index a0807a9..3254718 100644 --- a/musicxml-singer-with-oddvoices/src/App.tsx +++ b/musicxml-singer-with-oddvoices/src/App.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { forEach, map } from "lodash"; +import { map } from "lodash"; import { Accordion, AccordionDetails, @@ -10,32 +10,19 @@ import { Grid, Paper, Typography, - styled, } from "@mui/material"; -import CloudUploadIcon from "@mui/icons-material/CloudUpload"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; -import { SplitParams, createSplitOddVoiceJsonInputsFromMusicXml } from "./oddVoiceJSON"; -import { parseXmlText } from "./musicXmlParsing/xmlHelpers"; +import { SplitParams } from "./oddVoiceJSON"; import { OddVoiceJSON } from "./oddVoiceJSON/oddVoiceHelpers"; import { PhonemeGuide } from "./PhonemeGuide"; import { OpenSheetMusicDisplay } from "./OpenSheetMusicDisplay"; import { base64EncArr } from "./oddvoices/oddvoicesUtils"; import { useOddVoicesApp } from "./oddvoices"; +import { MediaControls } from "./MediaControls"; import "./App.css"; - -const VisuallyHiddenInput = styled("input")({ - clip: "rect(0 0 0 0)", - clipPath: "inset(50%)", - height: 1, - overflow: "hidden", - position: "absolute", - bottom: 0, - left: 0, - whiteSpace: "nowrap", - width: 1, -}); +import { UploadButton } from "./UploadButton"; function App() { const [rawFile, setRawFile] = React.useState(""); @@ -47,99 +34,75 @@ function App() { console.log({ oddVoiceOutputs }); } - const { oddVoiceApp, generateVoiceFromOddVoiceJson } = useOddVoicesApp(); + const { isLoadingApp, isLoadingVoice, generateVoiceFromOddVoiceJson } = useOddVoicesApp(); const [isGeneratingAudio, setIsGeneratingAudio] = React.useState(false); - const [, startTransition] = React.useTransition(); - return ( - {!oddVoiceApp ? ( + theme.palette.background.paper, + }} + > + : null}> + + {isGeneratingAudio ? ( + <> + Generating audio... + + ) : isLoadingVoice ? ( + <> + Loading voice... + + ) : rawFile ? ( + "View MusicXML" + ) : ( + "Upload a MusicXML file to view it here." + )} + + + + + + + + {isLoadingApp ? ( ) : ( <> - - - {audioOutputs.length > 0 && ( - - )} + {audioOutputs.length > 0 && } + - {isGeneratingAudio ? ( - - ) : ( - Boolean(rawFile) && ( - - }> - - View MusicXML - - - - - - - ) - )} - {!isGeneratingAudio && ( diff --git a/musicxml-singer-with-oddvoices/src/MediaControls.tsx b/musicxml-singer-with-oddvoices/src/MediaControls.tsx new file mode 100644 index 0000000..6570807 --- /dev/null +++ b/musicxml-singer-with-oddvoices/src/MediaControls.tsx @@ -0,0 +1,57 @@ +import React from "react"; +import { forEach } from "lodash"; +import { TextField, ToggleButton, ToggleButtonGroup } from "@mui/material"; +import PlayIcon from "@mui/icons-material/PlayArrow"; +import PauseIcon from "@mui/icons-material/Pause"; +import StopIcon from "@mui/icons-material/Stop"; + +export const MediaControls = () => { + const [jumpToTime, setJumpToTime] = React.useState(0); + return ( + { + const allAudios = document.querySelectorAll("audio"); + forEach(allAudios, (audio) => { + if (action === "play") { + audio.play(); + } else if (action === "pause") { + audio.pause(); + } else if (action === "stop") { + audio.pause(); + audio.currentTime = 0; + } else if (action === "jump") { + forEach(allAudios, (audio) => { + audio.currentTime = jumpToTime; + }); + } + }); + }} + aria-label="text formatting" + > + + + + + + + + + + {/* Jump to */} + + Jump to + + { + setJumpToTime(Number(e.target.value)); + }} + inputProps={{ min: 0, step: 1 }} + /> + + ); +}; diff --git a/musicxml-singer-with-oddvoices/src/UploadButton.tsx b/musicxml-singer-with-oddvoices/src/UploadButton.tsx new file mode 100644 index 0000000..e02ae4f --- /dev/null +++ b/musicxml-singer-with-oddvoices/src/UploadButton.tsx @@ -0,0 +1,88 @@ +import React from "react"; +import { forEach } from "lodash"; +import { Button, styled } from "@mui/material"; +import CloudUploadIcon from "@mui/icons-material/CloudUpload"; + +import { SplitParams, createSplitOddVoiceJsonInputsFromMusicXml } from "./oddVoiceJSON"; +import { parseXmlText } from "./musicXmlParsing/xmlHelpers"; +import { OddVoiceJSON } from "./oddVoiceJSON/oddVoiceHelpers"; + +const VisuallyHiddenInput = styled("input")({ + clip: "rect(0 0 0 0)", + clipPath: "inset(50%)", + height: 1, + overflow: "hidden", + position: "absolute", + bottom: 0, + left: 0, + whiteSpace: "nowrap", + width: 1, +}); + +export const UploadButton: React.FC<{ + isLoadingVoice: boolean; + setOddVoiceOutputs: React.Dispatch>>; + setAudioOutputs: React.Dispatch>; + setRawFile: React.Dispatch>; + setIsGeneratingAudio: React.Dispatch>; + generateVoiceFromOddVoiceJson: (oddVoiceJson: OddVoiceJSON) => Uint8Array | undefined; +}> = ({ + isLoadingVoice, + generateVoiceFromOddVoiceJson, + setIsGeneratingAudio, + setOddVoiceOutputs, + setAudioOutputs, + setRawFile, +}) => { + const [, startTransition] = React.useTransition(); + + return ( + + ); +}; diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/Makefile b/musicxml-singer-with-oddvoices/src/oddvoices/cpp/Makefile index 8abb30f..a42cd6a 100644 --- a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/Makefile +++ b/musicxml-singer-with-oddvoices/src/oddvoices/cpp/Makefile @@ -11,7 +11,7 @@ ODDVOICES_FRONTEND_SOURCES=oddvoices/cpp/frontend/sing.cpp oddvoices/cpp/fronten MIDIFILE_SOURCES=$(wildcard oddvoices/cpp/external_libraries/midifile/src/*.cpp) WASM_SOURCES=oddvoices_wasm.cpp ALL_CPP_SOURCES=$(WASM_SOURCES) $(ODDVOICES_SOURCES) $(ODDVOICES_FRONTEND_SOURCES) $(MIDIFILE_SOURCES) -PRELOAD_FILES=--preload-file oddvoices/cmudict-0.7b --preload-file voices/air.voice --preload-file voices/cicada.voice --preload-file voices/quake.voice +PRELOAD_FILES=--preload-file oddvoices/cmudict-0.7b # Determine the operating system UNAME_S := $(shell uname -s) diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/oddvoices_wasm.cpp b/musicxml-singer-with-oddvoices/src/oddvoices/cpp/oddvoices_wasm.cpp index 1374bf0..573962a 100644 --- a/musicxml-singer-with-oddvoices/src/oddvoices/cpp/oddvoices_wasm.cpp +++ b/musicxml-singer-with-oddvoices/src/oddvoices/cpp/oddvoices_wasm.cpp @@ -10,17 +10,11 @@ enum Voice QUAKE = 2 }; -std::string sing(int voiceIndex, std::string json, std::string outWAVFile, std::string lyrics) +std::string sing(oddvoices::Voice &voice, std::string json, std::string outWAVFile, std::string lyrics) { - // Setup the G2P Class oddvoices::g2p::G2P g2p("oddvoices/cmudict-0.7b"); - // Setup the Voice Class - std::string voicePath = voiceIndex == AIR ? "voices/air.voice" : voiceIndex == CICADA ? "voices/cicada.voice" : "voices/quake.voice"; - oddvoices::Voice voice; - voice.initFromFile(voicePath); - // Sing the JSON auto [ok, error] = oddvoices::frontend::singJSON( voice, g2p, json, outWAVFile, lyrics); @@ -34,4 +28,7 @@ std::string sing(int voiceIndex, std::string json, std::string outWAVFile, std:: EMSCRIPTEN_BINDINGS(oddvoices_wasm) { emscripten::function("sing", &sing); + emscripten::class_("Voice") + .constructor<>() + .function("initFromFile", &oddvoices::Voice::initFromFile); } diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/index.tsx b/musicxml-singer-with-oddvoices/src/oddvoices/index.tsx index cea0221..97e9665 100644 --- a/musicxml-singer-with-oddvoices/src/oddvoices/index.tsx +++ b/musicxml-singer-with-oddvoices/src/oddvoices/index.tsx @@ -3,14 +3,29 @@ import React from "react"; // @ts-expect-error - This is an autogenerated file that will be overwritten or replaced by the build process import createOddVoicesModule from "./js/oddvoices_wasm.mjs"; import { OddVoiceJSON } from "../oddVoiceJSON/oddVoiceHelpers"; -import { Voice } from "./oddvoicesUtils"; +import { Voice, voiceUrlPrefix } from "./oddvoicesUtils"; +import { useQuery } from "@tanstack/react-query"; + +interface VoiceObject { + initFromFile: (filename: string) => void; +} + +interface VoiceFactory { + new (): VoiceObject; +} export const useOddVoicesApp = () => { + const [didWriteVoicesFolder, setDidWriteVoicesFolder] = React.useState(false); + const [activeVoice, setActiveVoice] = React.useState(Voice.air); + const [oddVoiceApp, setOddVoiceApp] = React.useState<{ - sing: (voice: Voice, input: string, output: string, lyricsOverride: string) => string; + sing: (voice: VoiceObject, input: string, output: string, lyricsOverride: string) => string; FS: { readFile: (filename: string) => Uint8Array; - } + writeFile: (filename: string, data: Uint8Array) => void; + mkdir: (dirname: string) => void; + }; + Voice: VoiceFactory; } | null>(null); React.useEffect(() => { @@ -18,12 +33,48 @@ export const useOddVoicesApp = () => { initialize(); }, []); + React.useEffect(() => { + if (!oddVoiceApp) { + return; + } + oddVoiceApp?.FS.mkdir("/voices/"); + setDidWriteVoicesFolder(true); + }, [oddVoiceApp]); + + const { + data: voiceData, + isLoading: isLoadingVoice, + } = useQuery({ + queryKey: ["oddVoices", activeVoice], + queryFn: async () => { + if (!oddVoiceApp || !activeVoice) { + return; + } + const response = await fetch(`${voiceUrlPrefix}${activeVoice}.voice`); + const buffer = await response.arrayBuffer(); + + const fileName = `/voices/${activeVoice}.voice`; + oddVoiceApp.FS.writeFile(fileName, new Uint8Array(buffer)); + + const voice = new oddVoiceApp.Voice(); + voice.initFromFile(fileName); + + return voice; + }, + enabled: Boolean(didWriteVoicesFolder && activeVoice && oddVoiceApp), + retry: false, + }); + const generateVoiceFromOddVoiceJson = (oddVoiceJson: OddVoiceJSON): Uint8Array | undefined => { if (!oddVoiceApp) { console.error("OddVoice app not initialized"); return; } - const error: string = oddVoiceApp.sing(0, JSON.stringify(oddVoiceJson), "out.wav", ""); + if (!voiceData) { + console.error("Voice data not loaded"); + return; + } + const error: string = oddVoiceApp.sing(voiceData, JSON.stringify(oddVoiceJson), "out.wav", ""); if (error !== "") { console.error(error); return; @@ -38,7 +89,10 @@ export const useOddVoicesApp = () => { }; return { - oddVoiceApp, + isLoadingApp: !oddVoiceApp, + isLoadingVoice, generateVoiceFromOddVoiceJson, + activeVoice, + setActiveVoice, }; }; diff --git a/musicxml-singer-with-oddvoices/src/oddvoices/oddvoicesUtils.tsx b/musicxml-singer-with-oddvoices/src/oddvoices/oddvoicesUtils.tsx index 19a36e6..0e1ac7c 100644 --- a/musicxml-singer-with-oddvoices/src/oddvoices/oddvoicesUtils.tsx +++ b/musicxml-singer-with-oddvoices/src/oddvoices/oddvoicesUtils.tsx @@ -37,8 +37,10 @@ export function base64EncArr(aBytes: Uint8Array) { return sB64Enc.substring(0, sB64Enc.length - 2 + nMod3) + (nMod3 === 2 ? "" : nMod3 === 1 ? "=" : "=="); } +export const voiceUrlPrefix = "singing-synthesis/voices/"; + export enum Voice { - air=0, - cicada=1, - quake=2, + air = "air", + cicada = "cicada", + quake = "quake", }