now phasercanvas makes sounds used the same stack as previously and ram issue spotted

2023-07-25 16:04:08 +02:00
parent 7438986bcd
commit 7c3289ccec
3 changed files with 178 additions and 136 deletions
--- a/front/components/PartitionView.tsx
+++ b/front/components/PartitionView.tsx
@@ -27,7 +27,7 @@ type PartitionViewProps = {
 const PartitionView = (props: PartitionViewProps) => {
 	const [osmd, setOsmd] = useState<OSMD>();
 	const [soundPlayer, setSoundPlayer] = useState<SoundFont.Player>();
-	const audioContext = new SAC.AudioContext();
+	// const audioContext = new SAC.AudioContext();
 	// const [wholeNoteLength, setWholeNoteLength] = useState(0); // Length of Whole note, in ms (?)
 	const colorScheme = useColorScheme();
 	const dimensions = useWindowDimensions();
@@ -68,8 +68,8 @@ const PartitionView = (props: PartitionViewProps) => {
 			.filter((note) => note.Pitch) // Pitch Can be null, avoiding them
 			.forEach((note) => {
 				// Put your hands together for https://github.com/jimutt/osmd-audio-player/blob/master/src/internals/noteHelpers.ts
-				const fixedKey =
-					note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0;
+				// const fixedKey =
+				// 	note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0;
 				// const midiNumber = note.halfTone - fixedKey * 12;
 				// // console.log('Expecting midi ' + midiNumber);
 				// const duration = getActualNoteLength(note);
--- a/front/components/PartitionVisualizer/PhaserCanvas.tsx
+++ b/front/components/PartitionVisualizer/PhaserCanvas.tsx
@@ -6,14 +6,31 @@ import Phaser from 'phaser';
 import useColorScheme from '../../hooks/colorScheme';
 import { PartitionContext } from '../../views/PlayView';
 import { on } from 'events';
+import SoundFont from 'soundfont-player';
+import * as SAC from 'standardized-audio-context';

 let globalTimestamp = 0;
 let globalStatus: 'playing' | 'paused' | 'stopped' = 'playing';

+const playNotes = (notes: any[], soundPlayer: SoundFont.Player, audioContext: SAC.AudioContext) => {
+	notes.forEach(({ note, duration }) => {
+		const fixedKey =
+			note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0;
+		const midiNumber = note.halfTone - fixedKey * 12;
+		const gain = note.ParentVoiceEntry.ParentVoice.Volume;
+		soundPlayer!.play(midiNumber.toString(), audioContext.currentTime, {
+			duration,
+			gain,
+		});
+	});
+};
+
 const getPianoScene = (
 	partitionB64: string,
 	cursorPositions: PianoCursorPosition[],
 	onEndReached: () => void,
+	soundPlayer: SoundFont.Player,
+	audioContext: SAC.AudioContext,
 	colorScheme: 'light' | 'dark'
 ) => {
 	class PianoScene extends Phaser.Scene {
@@ -50,6 +67,7 @@ const getPianoScene = (
 					return false;
 				});
 				if (cP) {
+					playNotes(cP.notes, soundPlayer, audioContext);
 					const tw = {
 						targets: this!.cursor,
 						x: cP!.x,
@@ -58,6 +76,7 @@ const getPianoScene = (
 					};
 					if (this.cursorPositionsIdx === cursorPositions.length - 1) {
 						tw.onComplete = () => {
+							soundPlayer.stop();
 							onEndReached();
 						};
 					}
@@ -89,32 +108,43 @@ export type PhaserCanvasProps = {
 	onEndReached: () => void;
 };

-const PhaserCanvas = ({
-	partitionB64,
-	cursorPositions,
-	onEndReached,
-}: PhaserCanvasProps) => {
+const PhaserCanvas = ({ partitionB64, cursorPositions, onEndReached }: PhaserCanvasProps) => {
 	const colorScheme = useColorScheme();
+	const audioContext = new SAC.AudioContext();
+	const [soundPlayer, setSoundPlayer] = React.useState<SoundFont.Player>();
 	const { timestamp } = React.useContext(PartitionContext);
 	const [game, setGame] = React.useState<Phaser.Game | null>(null);

 	globalTimestamp = timestamp;
 	useEffect(() => {
-		const pianoScene = getPianoScene(partitionB64, cursorPositions, onEndReached, colorScheme);
+		Promise.resolve(
+			SoundFont.instrument(audioContext as unknown as AudioContext, 'electric_piano_1')
+		).then((sound) => {
+			setSoundPlayer(sound);

-		const config = {
-			type: Phaser.AUTO,
-			parent: 'phaser-canvas',
-			width: 1000,
-			height: 400,
-			scene: pianoScene,
-			scale: {
-				mode: Phaser.Scale.FIT,
-				autoCenter: Phaser.Scale.CENTER_BOTH,
-			},
-		};
+			const pianoScene = getPianoScene(
+				partitionB64,
+				cursorPositions,
+				onEndReached,
+				sound,
+				audioContext,
+				colorScheme
+			);

-		setGame(new Phaser.Game(config));
+			const config = {
+				type: Phaser.AUTO,
+				parent: 'phaser-canvas',
+				width: 1000,
+				height: 400,
+				scene: pianoScene,
+				scale: {
+					mode: Phaser.Scale.FIT,
+					autoCenter: Phaser.Scale.CENTER_BOTH,
+				},
+			};
+
+			setGame(new Phaser.Game(config));
+		});
 	}, []);

 	return <div id="phaser-canvas"></div>;
--- a/front/views/PlayView.tsx
+++ b/front/views/PlayView.tsx
@@ -76,7 +76,6 @@ export const PartitionContext = React.createContext<{
 	timestamp: 0,
 });

-
 const PlayView = ({ songId, type, route }: RouteProps<PlayViewProps>) => {
 	const accessToken = useSelector((state: RootState) => state.user.accessToken);
 	const navigation = useNavigation();
@@ -100,13 +99,15 @@ const PlayView = ({ songId, type, route }: RouteProps<PlayViewProps>) => {
 	const onPause = () => {
 		stopwatch.pause();
 		setPause(true);
-		webSocket.current?.send(
-			JSON.stringify({
-				type: 'pause',
-				paused: true,
-				time: getElapsedTime(),
-			})
-		);
+		if (webSocket.current?.readyState == WebSocket.OPEN) {
+			webSocket.current?.send(
+				JSON.stringify({
+					type: 'pause',
+					paused: true,
+					time: getElapsedTime(),
+				})
+			);
+		}
 	};
 	const onResume = () => {
 		if (stopwatch.isStarted()) {
@@ -115,20 +116,26 @@ const PlayView = ({ songId, type, route }: RouteProps<PlayViewProps>) => {
 			stopwatch.start();
 		}
 		setPause(false);
-		webSocket.current?.send(
-			JSON.stringify({
-				type: 'pause',
-				paused: false,
-				time: getElapsedTime(),
-			})
-		);
+		if (webSocket.current?.readyState == WebSocket.OPEN) {
+			webSocket.current?.send(
+				JSON.stringify({
+					type: 'pause',
+					paused: false,
+					time: getElapsedTime(),
+				})
+			);
+		}
 	};
 	const onEnd = () => {
-		webSocket.current?.send(
-			JSON.stringify({
-				type: 'end',
-			})
-		);
+		setTime(0);
+		stopwatch.stop();
+		if (webSocket.current?.readyState == WebSocket.OPEN) {
+			webSocket.current?.send(
+				JSON.stringify({
+					type: 'end',
+				})
+			);
+		}
 	};

 	const onMIDISuccess = (access: MIDIAccess) => {
@@ -224,7 +231,7 @@ const PlayView = ({ songId, type, route }: RouteProps<PlayViewProps>) => {
 	useEffect(() => {
 		ScreenOrientation.lockAsync(ScreenOrientation.OrientationLock.LANDSCAPE).catch(() => {});
 		const interval = setInterval(() => {
-			setTime(() =>getElapsedTime()); // Countdown
+			setTime(() => getElapsedTime()); // Countdown
 		}, 1);

 		return () => {
@@ -264,23 +271,23 @@ const PlayView = ({ songId, type, route }: RouteProps<PlayViewProps>) => {
 					timestamp: time,
 				}}
 			>
-			<HStack
-				width="100%"
-				justifyContent="center"
-				p={3}
-				style={{ position: 'absolute', top: 1 }}
-			>
-				<Animated.View style={{ opacity: fadeAnim }}>
-					<TextButton
-						disabled
-						label={lastScoreMessage?.content ?? ''}
-						colorScheme={lastScoreMessage?.color}
-						rounded="sm"
-					/>
-				</Animated.View>
-			</HStack>
-			<View style={{ flexGrow: 1, justifyContent: 'center' }}>
-				{/* <PartitionView
+				<HStack
+					width="100%"
+					justifyContent="center"
+					p={3}
+					style={{ position: 'absolute', top: 1 }}
+				>
+					<Animated.View style={{ opacity: fadeAnim }}>
+						<TextButton
+							disabled
+							label={lastScoreMessage?.content ?? ''}
+							colorScheme={lastScoreMessage?.color}
+							rounded="sm"
+						/>
+					</Animated.View>
+				</HStack>
+				<View style={{ flexGrow: 1, justifyContent: 'center' }}>
+					{/* <PartitionView
 					file={musixml.data}
 					onPartitionReady={() => setPartitionRendered(true)}
 					timestamp={Math.max(0, time)}
@@ -288,89 +295,94 @@ const PlayView = ({ songId, type, route }: RouteProps<PlayViewProps>) => {
 						onEnd();
 					}}
 				/> */}
-				<PartitionCoord
-					file={musixml.data}
-					// timestamp={Math.max(0, time)}
-					timestamp={0}
-					onEndReached={() => {
-						onEnd();
-					}}
-					onPartitionReady={() => setPartitionRendered(true)}
-				/>
-				{!partitionRendered && <LoadingComponent />}
-			</View>
-
-			<Box
-				shadow={4}
-				style={{
-					height: '12%',
-					width: '100%',
-					borderWidth: 0.5,
-					margin: 5,
-					display: !partitionRendered ? 'none' : undefined,
-				}}
-			>
-				<Row justifyContent="space-between" style={{ flexGrow: 1, alignItems: 'center' }}>
-					<Column
-						space={2}
-						style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}
-					>
-						<Text style={{ fontWeight: 'bold' }}>Score: {score}%</Text>
-						<Progress value={score} style={{ width: '90%' }} />
-					</Column>
-					<Center style={{ flex: 1, alignItems: 'center' }}>
-						<Text style={{ fontWeight: '700' }}>{song.data.name}</Text>
-					</Center>
-					<Row
-						style={{
-							flex: 1,
-							height: '100%',
-							justifyContent: 'space-evenly',
-							alignItems: 'center',
+					<PartitionCoord
+						file={musixml.data}
+						// timestamp={Math.max(0, time)}
+						timestamp={0}
+						onEndReached={() => {
+							onEnd();
 						}}
+						onPartitionReady={() => setPartitionRendered(true)}
+					/>
+					{!partitionRendered && <LoadingComponent />}
+				</View>
+
+				<Box
+					shadow={4}
+					style={{
+						height: '12%',
+						width: '100%',
+						borderWidth: 0.5,
+						margin: 5,
+						display: !partitionRendered ? 'none' : undefined,
+					}}
+				>
+					<Row
+						justifyContent="space-between"
+						style={{ flexGrow: 1, alignItems: 'center' }}
 					>
-						{midiKeyboardFound && (
-							<>
-								<IconButton
-									size="sm"
-									variant="solid"
-									icon={<Icon as={Ionicons} name={paused ? 'play' : 'pause'} />}
-									onPress={() => {
-										if (paused) {
-											onResume();
-										} else {
-											onPause();
+						<Column
+							space={2}
+							style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}
+						>
+							<Text style={{ fontWeight: 'bold' }}>Score: {score}%</Text>
+							<Progress value={score} style={{ width: '90%' }} />
+						</Column>
+						<Center style={{ flex: 1, alignItems: 'center' }}>
+							<Text style={{ fontWeight: '700' }}>{song.data.name}</Text>
+						</Center>
+						<Row
+							style={{
+								flex: 1,
+								height: '100%',
+								justifyContent: 'space-evenly',
+								alignItems: 'center',
+							}}
+						>
+							{midiKeyboardFound && (
+								<>
+									<IconButton
+										size="sm"
+										variant="solid"
+										icon={
+											<Icon as={Ionicons} name={paused ? 'play' : 'pause'} />
 										}
-									}}
-								/>
-								<Text>
-									{time < 0
-										? paused
-											? '0:00'
-											: Math.floor((time % 60000) / 1000)
+										onPress={() => {
+											if (paused) {
+												onResume();
+											} else {
+												onPause();
+											}
+										}}
+									/>
+									<Text>
+										{time < 0
+											? paused
+												? '0:00'
+												: Math.floor((time % 60000) / 1000)
+														.toFixed(0)
+														.toString()
+											: `${Math.floor(time / 60000)}:${Math.floor(
+													(time % 60000) / 1000
+											  )
 													.toFixed(0)
 													.toString()
-										: `${Math.floor(time / 60000)}:${Math.floor(
-												(time % 60000) / 1000
-										  )
-												.toFixed(0)
-												.toString()
-												.padStart(2, '0')}`}
-								</Text>
-								<IconButton
-									size="sm"
-									colorScheme="coolGray"
-									variant="solid"
-									icon={<Icon as={Ionicons} name="stop" />}
-									onPress={() => {
-										onEnd();
-									}}
-								/>
-							</>
-						)}
+													.padStart(2, '0')}`}
+									</Text>
+									<IconButton
+										size="sm"
+										colorScheme="coolGray"
+										variant="solid"
+										icon={<Icon as={Ionicons} name="stop" />}
+										onPress={() => {
+											onEnd();
+										}}
+									/>
+								</>
+							)}
+						</Row>
 					</Row>
-				</Row>
-			</Box>
+				</Box>
 			</PartitionContext.Provider>
 		</SafeAreaView>
 	);