Skip to content

Commit

Permalink
feat: separate voices from wasm package to hopefully be able to fix t…
Browse files Browse the repository at this point in the history
…he github pages issue, either via regular build or uploading a compiled app
  • Loading branch information
VehpuS committed Mar 7, 2024
1 parent 8d39ed4 commit fa5cd2b
Show file tree
Hide file tree
Showing 11 changed files with 275 additions and 114 deletions.
159 changes: 61 additions & 98 deletions musicxml-singer-with-oddvoices/src/App.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import React from "react";
import { forEach, map } from "lodash";
import { map } from "lodash";
import {
Accordion,
AccordionDetails,
Expand All @@ -10,32 +10,19 @@ import {
Grid,
Paper,
Typography,
styled,
} from "@mui/material";
import CloudUploadIcon from "@mui/icons-material/CloudUpload";
import ExpandMoreIcon from "@mui/icons-material/ExpandMore";

import { SplitParams, createSplitOddVoiceJsonInputsFromMusicXml } from "./oddVoiceJSON";
import { parseXmlText } from "./musicXmlParsing/xmlHelpers";
import { SplitParams } from "./oddVoiceJSON";
import { OddVoiceJSON } from "./oddVoiceJSON/oddVoiceHelpers";
import { PhonemeGuide } from "./PhonemeGuide";
import { OpenSheetMusicDisplay } from "./OpenSheetMusicDisplay";
import { base64EncArr } from "./oddvoices/oddvoicesUtils";
import { useOddVoicesApp } from "./oddvoices";
import { MediaControls } from "./MediaControls";

import "./App.css";

const VisuallyHiddenInput = styled("input")({
clip: "rect(0 0 0 0)",
clipPath: "inset(50%)",
height: 1,
overflow: "hidden",
position: "absolute",
bottom: 0,
left: 0,
whiteSpace: "nowrap",
width: 1,
});
import { UploadButton } from "./UploadButton";

function App() {
const [rawFile, setRawFile] = React.useState<string>("");
Expand All @@ -47,99 +34,75 @@ function App() {
console.log({ oddVoiceOutputs });
}

const { oddVoiceApp, generateVoiceFromOddVoiceJson } = useOddVoicesApp();
const { isLoadingApp, isLoadingVoice, generateVoiceFromOddVoiceJson } = useOddVoicesApp();

const [isGeneratingAudio, setIsGeneratingAudio] = React.useState(false);

const [, startTransition] = React.useTransition();

return (
<Paper>
{!oddVoiceApp ? (
<Accordion
variant="outlined"
sx={{
top: 0,
position: "sticky",
zIndex: 1,
backgroundColor: (theme) => theme.palette.background.paper,
}}
>
<AccordionSummary disabled={!rawFile} expandIcon={rawFile ? <ExpandMoreIcon /> : null}>
<Typography variant="body1" textAlign="center" width="100%">
{isGeneratingAudio ? (
<>
<CircularProgress size={16} /> Generating audio...
</>
) : isLoadingVoice ? (
<>
<CircularProgress size={16} /> Loading voice...
</>
) : rawFile ? (
"View MusicXML"
) : (
"Upload a MusicXML file to view it here."
)}
</Typography>
</AccordionSummary>
<AccordionDetails sx={{ overflow: "auto", height: 300 }}>
<OpenSheetMusicDisplay autoResize file={rawFile} />
</AccordionDetails>
</Accordion>

{isLoadingApp ? (
<CircularProgress />
) : (
<>
<Grid item>
<Button
component="label"
role={undefined}
variant="contained"
tabIndex={-1}
startIcon={<CloudUploadIcon />}
<Grid
item
sx={{
top: 50,
position: "sticky",
zIndex: 1,
}}
>
<Paper
elevation={0}
sx={{
display: "flex",
border: (theme) => `1px solid ${theme.palette.divider}`,
flexWrap: "wrap",
}}
>
Upload file
<VisuallyHiddenInput
type="file"
onChange={(e) => {
const file = e?.target?.files?.[0];
if (!file) {
return;
}
const reader = new FileReader();
reader.onload = (e) => {
if (e?.target?.result) {
setIsGeneratingAudio(true);
const result = e.target.result as string;
startTransition(() => {
const newOddVoiceOutputs = createSplitOddVoiceJsonInputsFromMusicXml(
parseXmlText(result)
);
setOddVoiceOutputs(newOddVoiceOutputs);
setRawFile(result);
forEach(newOddVoiceOutputs, (oddVoiceOutput, i) => {
const outputAudio = generateVoiceFromOddVoiceJson(
oddVoiceOutput.output
);
if (!outputAudio || outputAudio.length === 0) {
console.error("Failed to generate audio output.");
return;
}
setAudioOutputs((prev) => {
const newOutputs = [...prev];
newOutputs[i] = outputAudio;
return newOutputs;
});
});
setIsGeneratingAudio(false);
});
}
};
reader.readAsText(file);
}}
<UploadButton
isLoadingVoice={isLoadingVoice}
setIsGeneratingAudio={setIsGeneratingAudio}
setOddVoiceOutputs={setOddVoiceOutputs}
setAudioOutputs={setAudioOutputs}
setRawFile={setRawFile}
generateVoiceFromOddVoiceJson={generateVoiceFromOddVoiceJson}
/>
</Button>
{audioOutputs.length > 0 && (
<Button
onClick={() => {
const allAudios = document.querySelectorAll("audio");
forEach(allAudios, (audio) => {
audio.currentTime = 0;
audio.play();
});
}}
>
Play All
</Button>
)}
{audioOutputs.length > 0 && <MediaControls />}
</Paper>
</Grid>
<Divider />
{isGeneratingAudio ? (
<CircularProgress />
) : (
Boolean(rawFile) && (
<Accordion variant="outlined">
<AccordionSummary expandIcon={<ExpandMoreIcon />}>
<Typography variant="body1" textAlign="center" width="100%">
View MusicXML
</Typography>
</AccordionSummary>
<AccordionDetails>
<OpenSheetMusicDisplay autoResize file={rawFile} />
</AccordionDetails>
</Accordion>
)
)}
<Divider />
{!isGeneratingAudio && (
<Grid container direction="column" gap={3} alignItems="center" paddingBlock={2}>
<Grid item container direction="column" gap={3} alignItems="center" alignSelf="center">
Expand Down
57 changes: 57 additions & 0 deletions musicxml-singer-with-oddvoices/src/MediaControls.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import React from "react";
import { forEach } from "lodash";
import { TextField, ToggleButton, ToggleButtonGroup } from "@mui/material";
import PlayIcon from "@mui/icons-material/PlayArrow";
import PauseIcon from "@mui/icons-material/Pause";
import StopIcon from "@mui/icons-material/Stop";

export const MediaControls = () => {
const [jumpToTime, setJumpToTime] = React.useState(0);
return (
<ToggleButtonGroup
size="small"
exclusive
onChange={(_event, action) => {
const allAudios = document.querySelectorAll("audio");
forEach(allAudios, (audio) => {
if (action === "play") {
audio.play();
} else if (action === "pause") {
audio.pause();
} else if (action === "stop") {
audio.pause();
audio.currentTime = 0;
} else if (action === "jump") {
forEach(allAudios, (audio) => {
audio.currentTime = jumpToTime;
});
}
});
}}
aria-label="text formatting"
>
<ToggleButton value="play" aria-label="italic">
<PlayIcon />
</ToggleButton>
<ToggleButton value="pause" aria-label="underlined">
<PauseIcon />
</ToggleButton>
<ToggleButton value="stop" aria-label="color">
<StopIcon />
</ToggleButton>
{/* Jump to */}
<ToggleButton value="jump" aria-label="color">
Jump to
</ToggleButton>
<TextField
type="number"
value={jumpToTime}
placeholder="Enter time in seconds"
onChange={(e) => {
setJumpToTime(Number(e.target.value));
}}
inputProps={{ min: 0, step: 1 }}
/>
</ToggleButtonGroup>
);
};
88 changes: 88 additions & 0 deletions musicxml-singer-with-oddvoices/src/UploadButton.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import React from "react";
import { forEach } from "lodash";
import { Button, styled } from "@mui/material";
import CloudUploadIcon from "@mui/icons-material/CloudUpload";

import { SplitParams, createSplitOddVoiceJsonInputsFromMusicXml } from "./oddVoiceJSON";
import { parseXmlText } from "./musicXmlParsing/xmlHelpers";
import { OddVoiceJSON } from "./oddVoiceJSON/oddVoiceHelpers";

const VisuallyHiddenInput = styled("input")({
clip: "rect(0 0 0 0)",
clipPath: "inset(50%)",
height: 1,
overflow: "hidden",
position: "absolute",
bottom: 0,
left: 0,
whiteSpace: "nowrap",
width: 1,
});

export const UploadButton: React.FC<{
isLoadingVoice: boolean;
setOddVoiceOutputs: React.Dispatch<React.SetStateAction<Array<{ output: OddVoiceJSON; splitParams: SplitParams }>>>;
setAudioOutputs: React.Dispatch<React.SetStateAction<Uint8Array[]>>;
setRawFile: React.Dispatch<React.SetStateAction<string>>;
setIsGeneratingAudio: React.Dispatch<React.SetStateAction<boolean>>;
generateVoiceFromOddVoiceJson: (oddVoiceJson: OddVoiceJSON) => Uint8Array | undefined;
}> = ({
isLoadingVoice,
generateVoiceFromOddVoiceJson,
setIsGeneratingAudio,
setOddVoiceOutputs,
setAudioOutputs,
setRawFile,
}) => {
const [, startTransition] = React.useTransition();

return (
<Button
component="label"
role={undefined}
variant="contained"
tabIndex={-1}
startIcon={<CloudUploadIcon />}
disabled={isLoadingVoice}
>
Upload file
<VisuallyHiddenInput
type="file"
onChange={(e) => {
const file = e?.target?.files?.[0];
if (!file) {
return;
}
const reader = new FileReader();
reader.onload = (e) => {
if (e?.target?.result) {
setIsGeneratingAudio(true);
const result = e.target.result as string;
startTransition(() => {
const newOddVoiceOutputs = createSplitOddVoiceJsonInputsFromMusicXml(
parseXmlText(result)
);
setOddVoiceOutputs(newOddVoiceOutputs);
setRawFile(result);
forEach(newOddVoiceOutputs, (oddVoiceOutput, i) => {
const outputAudio = generateVoiceFromOddVoiceJson(oddVoiceOutput.output);
if (!outputAudio || outputAudio.length === 0) {
console.error("Failed to generate audio output.");
return;
}
setAudioOutputs((prev) => {
const newOutputs = [...prev];
newOutputs[i] = outputAudio;
return newOutputs;
});
});
setIsGeneratingAudio(false);
});
}
};
reader.readAsText(file);
}}
/>
</Button>
);
};
2 changes: 1 addition & 1 deletion musicxml-singer-with-oddvoices/src/oddvoices/cpp/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ ODDVOICES_FRONTEND_SOURCES=oddvoices/cpp/frontend/sing.cpp oddvoices/cpp/fronten
MIDIFILE_SOURCES=$(wildcard oddvoices/cpp/external_libraries/midifile/src/*.cpp)
WASM_SOURCES=oddvoices_wasm.cpp
ALL_CPP_SOURCES=$(WASM_SOURCES) $(ODDVOICES_SOURCES) $(ODDVOICES_FRONTEND_SOURCES) $(MIDIFILE_SOURCES)
PRELOAD_FILES=--preload-file oddvoices/cmudict-0.7b --preload-file voices/air.voice --preload-file voices/cicada.voice --preload-file voices/quake.voice
PRELOAD_FILES=--preload-file oddvoices/cmudict-0.7b

# Determine the operating system
UNAME_S := $(shell uname -s)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,11 @@ enum Voice
QUAKE = 2
};

std::string sing(int voiceIndex, std::string json, std::string outWAVFile, std::string lyrics)
std::string sing(oddvoices::Voice &voice, std::string json, std::string outWAVFile, std::string lyrics)
{

// Setup the G2P Class
oddvoices::g2p::G2P g2p("oddvoices/cmudict-0.7b");

// Setup the Voice Class
std::string voicePath = voiceIndex == AIR ? "voices/air.voice" : voiceIndex == CICADA ? "voices/cicada.voice" : "voices/quake.voice";
oddvoices::Voice voice;
voice.initFromFile(voicePath);

// Sing the JSON
auto [ok, error] = oddvoices::frontend::singJSON(
voice, g2p, json, outWAVFile, lyrics);
Expand All @@ -34,4 +28,7 @@ std::string sing(int voiceIndex, std::string json, std::string outWAVFile, std::
EMSCRIPTEN_BINDINGS(oddvoices_wasm)
{
emscripten::function("sing", &sing);
emscripten::class_<oddvoices::Voice>("Voice")
.constructor<>()
.function("initFromFile", &oddvoices::Voice::initFromFile);
}
Loading

0 comments on commit fa5cd2b

Please sign in to comment.