using smplr player on web and mp3 on mobile

front/components/Metronome.tsx

@@ -63,7 +63,7 @@ export const MetronomeControls = ({ paused = false, bpm }: { paused?: boolean; b
 					}
 					onPress={() => setEnabled(!enabled)}
 				/>
-				{/* <Slider
+				<Slider
 					maxWidth={'500px'}
 					flex={1}
 					defaultValue={volume.current}
@@ -73,7 +73,7 @@ export const MetronomeControls = ({ paused = false, bpm }: { paused?: boolean; b
 						<Slider.FilledTrack />
 					</Slider.Track>
 					<Slider.Thumb />
-				</Slider> */}
+				</Slider>
 			</View>
 		</View>
 	);

front/components/Play/PartitionMagic.tsx

@@ -1,5 +1,5 @@
 import * as React from 'react';
-import { View } from 'react-native';
+import { Platform, View } from 'react-native';
 import API from '../../API';
 import { useQuery } from '../../Queries';
 import Animated, { useSharedValue, withTiming, Easing } from 'react-native-reanimated';
@@ -7,9 +7,10 @@ import { CursorInfoItem } from '../../models/SongCursorInfos';
 import { Audio } from 'expo-av';
 import { SvgContainer } from './SvgContainer';
 import LoadingComponent from '../Loading';
+import { SplendidGrandPiano } from 'smplr';
 
-// note we are also using timestamp in a context
 export type ParitionMagicProps = {
+	timestamp: number;
 	songID: number;
 	shouldPlay: boolean;
 	onEndReached: () => void;
@@ -43,6 +44,7 @@ const getCursorToPlay = (
 const transitionDuration = 50;
 
 const PartitionMagic = ({
+	timestamp,
 	songID,
 	shouldPlay,
 	onEndReached,
@@ -57,6 +59,8 @@ const PartitionMagic = ({
 	const [isPartitionSvgLoaded, setIsPartitionSvgLoaded] = React.useState(false);
 	const partitionOffset = useSharedValue(0);
 	const melodySound = React.useRef<Audio.Sound | null>(null);
+	const piano = React.useRef<SplendidGrandPiano | null>(null);
+	const [isPianoLoaded, setIsPianoLoaded] = React.useState(false);
 	const cursorPaddingVertical = 10;
 	const cursorPaddingHorizontal = 3;
 
@@ -75,12 +79,21 @@ const PartitionMagic = ({
 	}
 
 	React.useEffect(() => {
-		if (!melodySound.current) {
-			Audio.Sound.createAsync({
+		if (Platform.OS === 'web' && !piano.current) {
+			const audio = new AudioContext();
+			piano.current = new SplendidGrandPiano(audio);
+			piano.current.load.then(() => {
+				setIsPianoLoaded(true);
+			});
+		} else if (!melodySound.current) {
+			Audio.Sound.createAsync(
+				{
 					uri: API.getPartitionMelodyUrl(songID),
-			}, {
+				},
+				{
 					progressUpdateIntervalMillis: 200,
-			}).then((track) => {
+				}
+			).then((track) => {
 				melodySound.current = track.sound;
 			});
 		}
@@ -89,6 +102,11 @@ const PartitionMagic = ({
 				melodySound.current.pauseAsync();
 				melodySound.current.unloadAsync();
 			}
+			if (piano.current) {
+				piano.current.stop();
+				piano.current.context.close();
+				piano.current = null;
+			}
 		};
 	}, []);
 	const partitionDims = React.useMemo<[number, number]>(() => {
@@ -103,12 +121,19 @@ const PartitionMagic = ({
 	}, [onError, isError]);
 
 	React.useEffect(() => {
-		if (isPartitionSvgLoaded && !isLoading && melodySound.current?._loaded) {
+		if (isPartitionSvgLoaded && !isLoading && (melodySound.current?._loaded || isPianoLoaded)) {
 			onReady();
 		}
-	}, [isPartitionSvgLoaded, isLoading, melodySound.current?._loaded]);
+	}, [isPartitionSvgLoaded, isLoading, melodySound.current?._loaded, isPianoLoaded]);
 
 	React.useEffect(() => {
+		if (Platform.OS === 'web') {
+			if (!piano.current || !isPianoLoaded) {
+				return;
+			}
+			shouldPlay ? onPlay() : onPause();
+			return;
+		}
 		if (!melodySound.current || !melodySound.current._loaded) {
 			return;
 		}
@@ -116,6 +141,7 @@ const PartitionMagic = ({
 			melodySound.current.getStatusAsync().then((status) => {
 				const lastCur = data!.cursors[data!.cursors.length - 1]!;
 				const maxTs = lastCur.timestamp + lastCur.timing;
+				//@ts-expect-error error in the type
 				const newRate = status.durationMillis! / maxTs;
 				console.log('newRate', newRate);
 				if (newRate < 0 || newRate > 2) {
@@ -140,7 +166,7 @@ const PartitionMagic = ({
 
 	React.useEffect(() => {
 		if (!melodySound.current || !melodySound.current._loaded) return;
-		if (data?.cursors.length === 0) return;
+		if (!data || data?.cursors.length === 0) return;
 
 		melodySound.current.setOnPlaybackStatusUpdate((status) => {
 			//@ts-expect-error positionMillis is not in the type
@@ -164,6 +190,34 @@ const PartitionMagic = ({
 		});
 	}, [data?.cursors, melodySound.current?._loaded]);
 
+	React.useEffect(() => {
+		if (!shouldPlay) return;
+		if (!piano.current || !isPianoLoaded) return;
+		if (!data || data?.cursors.length === 0) return;
+		getCursorToPlay(
+			data!.cursors,
+			currentCurIdx.current,
+			timestamp + transitionDuration,
+			(cursor, idx) => {
+				console.log('cursor', cursor);
+				currentCurIdx.current = idx;
+				partitionOffset.value = withTiming(
+					-(cursor.x - data!.cursors[0]!.x) / partitionDims[0],
+					{
+						duration: transitionDuration,
+						easing: Easing.inOut(Easing.ease),
+					}
+				);
+				cursor.notes.forEach((note) => {
+					piano.current?.start({
+						note: note.note,
+						duration: note.duration,
+					});
+				});
+			}
+		);
+	}, [timestamp, data?.cursors, isPianoLoaded]);
+
 	return (
 		<View
 			style={{

front/components/Play/PlayViewControlBar.tsx

@@ -1,7 +1,6 @@
 import { View } from 'react-native';
 import * as React from 'react';
-import { Row, Image, Text, Icon, useBreakpointValue, IconButton } from 'native-base';
-// import IconButton from '../IconButton';
+import { Row, Image, Text, useBreakpointValue, IconButton } from 'native-base';
 import { Ionicons } from '@expo/vector-icons';
 import { MetronomeControls } from '../Metronome';
 import StarProgress from '../StarProgress';

front/package.json

@@ -60,6 +60,7 @@
 		"react-redux": "^8.1.2",
 		"react-use-precision-timer": "^3.3.1",
 		"redux-persist": "^6.0.0",
+		"smplr": "^0.12.1",
 		"type-fest": "^4.3.2",
 		"url": "^0.11.3",
 		"yup": "^1.3.1"

front/views/PlayView.tsx

@@ -387,7 +387,7 @@ const PlayView = ({ songId }: PlayViewProps) => {
 				>
 					<PartitionMagic
 						shouldPlay={shouldPlay}
-						// timestamp={time}
+						timestamp={time}
 						songID={song.data.id}
 						onEndReached={() => {
 							setTimeout(() => {

front/yarn.lock

@@ -10764,6 +10764,11 @@ slugify@^1.3.4:
   resolved "https://registry.yarnpkg.com/slugify/-/slugify-1.6.6.tgz#2d4ac0eacb47add6af9e04d3be79319cbcc7924b"
   integrity sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==
 
+smplr@^0.12.1:
+  version "0.12.1"
+  resolved "https://registry.yarnpkg.com/smplr/-/smplr-0.12.1.tgz#53895a6928c2a3093d72e639440ca4be0b480e54"
+  integrity sha512-GEHuuP2vCtgu7+HY2F0DEnIxB5JHfxsZt4lJd9i/zxiZz1eBHheiy9h5Th32XGkLGObm/MMCfVs8Fwz6N/BKMw==
+
 sockjs@^0.3.24:
   version "0.3.24"
   resolved "https://registry.yarnpkg.com/sockjs/-/sockjs-0.3.24.tgz#c9bc8995f33a111bea0395ec30aa3206bdb5ccce"