diff --git a/front/components/PartitionView.tsx b/front/components/PartitionView.tsx index 20e3781..765f9ae 100644 --- a/front/components/PartitionView.tsx +++ b/front/components/PartitionView.tsx @@ -27,7 +27,7 @@ type PartitionViewProps = { const PartitionView = (props: PartitionViewProps) => { const [osmd, setOsmd] = useState(); const [soundPlayer, setSoundPlayer] = useState(); - const audioContext = new SAC.AudioContext(); + // const audioContext = new SAC.AudioContext(); // const [wholeNoteLength, setWholeNoteLength] = useState(0); // Length of Whole note, in ms (?) const colorScheme = useColorScheme(); const dimensions = useWindowDimensions(); @@ -68,8 +68,8 @@ const PartitionView = (props: PartitionViewProps) => { .filter((note) => note.Pitch) // Pitch Can be null, avoiding them .forEach((note) => { // Put your hands together for https://github.com/jimutt/osmd-audio-player/blob/master/src/internals/noteHelpers.ts - const fixedKey = - note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0; + // const fixedKey = + // note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0; // const midiNumber = note.halfTone - fixedKey * 12; // // console.log('Expecting midi ' + midiNumber); // const duration = getActualNoteLength(note); diff --git a/front/components/PartitionVisualizer/PhaserCanvas.tsx b/front/components/PartitionVisualizer/PhaserCanvas.tsx index dccc9b2..51fbbf7 100644 --- a/front/components/PartitionVisualizer/PhaserCanvas.tsx +++ b/front/components/PartitionVisualizer/PhaserCanvas.tsx @@ -6,14 +6,31 @@ import Phaser from 'phaser'; import useColorScheme from '../../hooks/colorScheme'; import { PartitionContext } from '../../views/PlayView'; import { on } from 'events'; +import SoundFont from 'soundfont-player'; +import * as SAC from 'standardized-audio-context'; let globalTimestamp = 0; let globalStatus: 'playing' | 'paused' | 'stopped' = 'playing'; +const playNotes = (notes: any[], soundPlayer: SoundFont.Player, audioContext: SAC.AudioContext) => { + notes.forEach(({ note, duration }) => { + const fixedKey = + note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0; + const midiNumber = note.halfTone - fixedKey * 12; + const gain = note.ParentVoiceEntry.ParentVoice.Volume; + soundPlayer!.play(midiNumber.toString(), audioContext.currentTime, { + duration, + gain, + }); + }); +}; + const getPianoScene = ( partitionB64: string, cursorPositions: PianoCursorPosition[], onEndReached: () => void, + soundPlayer: SoundFont.Player, + audioContext: SAC.AudioContext, colorScheme: 'light' | 'dark' ) => { class PianoScene extends Phaser.Scene { @@ -50,6 +67,7 @@ const getPianoScene = ( return false; }); if (cP) { + playNotes(cP.notes, soundPlayer, audioContext); const tw = { targets: this!.cursor, x: cP!.x, @@ -58,6 +76,7 @@ const getPianoScene = ( }; if (this.cursorPositionsIdx === cursorPositions.length - 1) { tw.onComplete = () => { + soundPlayer.stop(); onEndReached(); }; } @@ -89,32 +108,43 @@ export type PhaserCanvasProps = { onEndReached: () => void; }; -const PhaserCanvas = ({ - partitionB64, - cursorPositions, - onEndReached, -}: PhaserCanvasProps) => { +const PhaserCanvas = ({ partitionB64, cursorPositions, onEndReached }: PhaserCanvasProps) => { const colorScheme = useColorScheme(); + const audioContext = new SAC.AudioContext(); + const [soundPlayer, setSoundPlayer] = React.useState(); const { timestamp } = React.useContext(PartitionContext); const [game, setGame] = React.useState(null); globalTimestamp = timestamp; useEffect(() => { - const pianoScene = getPianoScene(partitionB64, cursorPositions, onEndReached, colorScheme); + Promise.resolve( + SoundFont.instrument(audioContext as unknown as AudioContext, 'electric_piano_1') + ).then((sound) => { + setSoundPlayer(sound); - const config = { - type: Phaser.AUTO, - parent: 'phaser-canvas', - width: 1000, - height: 400, - scene: pianoScene, - scale: { - mode: Phaser.Scale.FIT, - autoCenter: Phaser.Scale.CENTER_BOTH, - }, - }; + const pianoScene = getPianoScene( + partitionB64, + cursorPositions, + onEndReached, + sound, + audioContext, + colorScheme + ); - setGame(new Phaser.Game(config)); + const config = { + type: Phaser.AUTO, + parent: 'phaser-canvas', + width: 1000, + height: 400, + scene: pianoScene, + scale: { + mode: Phaser.Scale.FIT, + autoCenter: Phaser.Scale.CENTER_BOTH, + }, + }; + + setGame(new Phaser.Game(config)); + }); }, []); return
; diff --git a/front/views/PlayView.tsx b/front/views/PlayView.tsx index f85e044..63e7afc 100644 --- a/front/views/PlayView.tsx +++ b/front/views/PlayView.tsx @@ -76,7 +76,6 @@ export const PartitionContext = React.createContext<{ timestamp: 0, }); - const PlayView = ({ songId, type, route }: RouteProps) => { const accessToken = useSelector((state: RootState) => state.user.accessToken); const navigation = useNavigation(); @@ -100,13 +99,15 @@ const PlayView = ({ songId, type, route }: RouteProps) => { const onPause = () => { stopwatch.pause(); setPause(true); - webSocket.current?.send( - JSON.stringify({ - type: 'pause', - paused: true, - time: getElapsedTime(), - }) - ); + if (webSocket.current?.readyState == WebSocket.OPEN) { + webSocket.current?.send( + JSON.stringify({ + type: 'pause', + paused: true, + time: getElapsedTime(), + }) + ); + } }; const onResume = () => { if (stopwatch.isStarted()) { @@ -115,20 +116,26 @@ const PlayView = ({ songId, type, route }: RouteProps) => { stopwatch.start(); } setPause(false); - webSocket.current?.send( - JSON.stringify({ - type: 'pause', - paused: false, - time: getElapsedTime(), - }) - ); + if (webSocket.current?.readyState == WebSocket.OPEN) { + webSocket.current?.send( + JSON.stringify({ + type: 'pause', + paused: false, + time: getElapsedTime(), + }) + ); + } }; const onEnd = () => { - webSocket.current?.send( - JSON.stringify({ - type: 'end', - }) - ); + setTime(0); + stopwatch.stop(); + if (webSocket.current?.readyState == WebSocket.OPEN) { + webSocket.current?.send( + JSON.stringify({ + type: 'end', + }) + ); + } }; const onMIDISuccess = (access: MIDIAccess) => { @@ -224,7 +231,7 @@ const PlayView = ({ songId, type, route }: RouteProps) => { useEffect(() => { ScreenOrientation.lockAsync(ScreenOrientation.OrientationLock.LANDSCAPE).catch(() => {}); const interval = setInterval(() => { - setTime(() =>getElapsedTime()); // Countdown + setTime(() => getElapsedTime()); // Countdown }, 1); return () => { @@ -264,23 +271,23 @@ const PlayView = ({ songId, type, route }: RouteProps) => { timestamp: time, }} > - - - - - - - {/* + + + + + + {/* setPartitionRendered(true)} timestamp={Math.max(0, time)} @@ -288,89 +295,94 @@ const PlayView = ({ songId, type, route }: RouteProps) => { onEnd(); }} /> */} - { - onEnd(); - }} - onPartitionReady={() => setPartitionRendered(true)} - /> - {!partitionRendered && } - - - - - - Score: {score}% - - -
- {song.data.name} -
- { + onEnd(); }} + onPartitionReady={() => setPartitionRendered(true)} + /> + {!partitionRendered && } +
+ + + - {midiKeyboardFound && ( - <> - } - onPress={() => { - if (paused) { - onResume(); - } else { - onPause(); + + Score: {score}% + + +
+ {song.data.name} +
+ + {midiKeyboardFound && ( + <> + } - }} - /> - - {time < 0 - ? paused - ? '0:00' - : Math.floor((time % 60000) / 1000) + onPress={() => { + if (paused) { + onResume(); + } else { + onPause(); + } + }} + /> + + {time < 0 + ? paused + ? '0:00' + : Math.floor((time % 60000) / 1000) + .toFixed(0) + .toString() + : `${Math.floor(time / 60000)}:${Math.floor( + (time % 60000) / 1000 + ) .toFixed(0) .toString() - : `${Math.floor(time / 60000)}:${Math.floor( - (time % 60000) / 1000 - ) - .toFixed(0) - .toString() - .padStart(2, '0')}`} - - } - onPress={() => { - onEnd(); - }} - /> - - )} + .padStart(2, '0')}`} + + } + onPress={() => { + onEnd(); + }} + /> + + )} +
- -
+ );