diff --git a/front/components/PartitionCoord.tsx b/front/components/PartitionCoord.tsx new file mode 100644 index 0000000..eb5eac3 --- /dev/null +++ b/front/components/PartitionCoord.tsx @@ -0,0 +1,40 @@ +import * as React from 'react'; +import PartitionView from './PartitionView'; +import PhaserCanvas from './PartitionVisualizer/PhaserCanvas'; + +type PartitionCoordProps = { + // The Buffer of the MusicXML file retreived from the API + file: string; + onPartitionReady: () => void; + onEndReached: () => void; + // Timestamp of the play session, in milisecond + timestamp: number; +}; + +const PartitionCoord = ({ + file, + onPartitionReady, + onEndReached, + timestamp, +}: PartitionCoordProps) => { + const [partitionB64, setPartitionB64] = React.useState(null); + + return ( + <> + {!partitionB64 && ( + { + setPartitionB64(base64data); + onPartitionReady(); + }} + onEndReached={onEndReached} + timestamp={timestamp} + /> + )} + {partitionB64 && } + + ); +}; + +export default PartitionCoord; \ No newline at end of file diff --git a/front/components/PartitionView.tsx b/front/components/PartitionView.tsx index b162fa6..1eed6b1 100644 --- a/front/components/PartitionView.tsx +++ b/front/components/PartitionView.tsx @@ -17,7 +17,7 @@ import * as SAC from 'standardized-audio-context'; type PartitionViewProps = { // The Buffer of the MusicXML file retreived from the API file: string; - onPartitionReady: () => void; + onPartitionReady: (base64data: string) => void; onEndReached: () => void; // Timestamp of the play session, in milisecond timestamp: number; @@ -33,6 +33,7 @@ const PartitionView = (props: PartitionViewProps) => { const OSMD_DIV_ID = 'osmd-div'; const options: IOSMDOptions = { darkMode: colorScheme == 'dark', + backend: 'canvas', drawComposer: false, drawCredits: false, drawLyrics: false, @@ -68,14 +69,14 @@ const PartitionView = (props: PartitionViewProps) => { // Put your hands together for https://github.com/jimutt/osmd-audio-player/blob/master/src/internals/noteHelpers.ts const fixedKey = note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0; - const midiNumber = note.halfTone - fixedKey * 12; - // console.log('Expecting midi ' + midiNumber); - const duration = getActualNoteLength(note); - const gain = note.ParentVoiceEntry.ParentVoice.Volume; - soundPlayer!.play(midiNumber.toString(), audioContext.currentTime, { - duration, - gain, - }); + // const midiNumber = note.halfTone - fixedKey * 12; + // // console.log('Expecting midi ' + midiNumber); + // const duration = getActualNoteLength(note); + // const gain = note.ParentVoiceEntry.ParentVoice.Volume; + // soundPlayer!.play(midiNumber.toString(), audioContext.currentTime, { + // duration, + // gain, + // }); }); }; const getShortedNoteUnderCursor = () => { @@ -88,16 +89,29 @@ const PartitionView = (props: PartitionViewProps) => { useEffect(() => { const _osmd = new OSMD(OSMD_DIV_ID, options); Promise.all([ - SoundFont.instrument(audioContext as unknown as AudioContext, 'electric_piano_1'), + // SoundFont.instrument(audioContext as unknown as AudioContext, 'electric_piano_1'), _osmd.load(props.file), ]).then(([player]) => { - setSoundPlayer(player); + // setSoundPlayer(player); _osmd.render(); + _osmd.cursor.show(); + // get the current cursor position + const curPos = []; + while (!_osmd.cursor.iterator.EndReached) { + curPos.push(_osmd.cursor.cursorElement.offsetLeft); + _osmd.cursor.next(); + } + console.log('curPos', curPos); + _osmd.cursor.reset(); _osmd.cursor.hide(); + console.log("timestamp cursor", _osmd.cursor.iterator.CurrentSourceTimestamp); + console.log("timestamp cursor", _osmd.cursor.iterator.CurrentVoiceEntries); + console.log("current measure index", _osmd.cursor.iterator.CurrentMeasureIndex); + const osmdCanvas = document.querySelector( "#" + OSMD_DIV_ID + " canvas"); // Ty https://github.com/jimutt/osmd-audio-player/blob/ec205a6e46ee50002c1fa8f5999389447bba7bbf/src/PlaybackEngine.ts#LL77C12-L77C63 const bpm = _osmd.Sheet.HasBPMInfo ? _osmd.Sheet.getExpressionsStartTempoInBPM() : 60; setWholeNoteLength(Math.round((60 / bpm) * 4000)); - props.onPartitionReady(); + props.onPartitionReady(osmdCanvas.toDataURL()); // Do not show cursor before actuall start }); setOsmd(_osmd); @@ -114,7 +128,7 @@ const PartitionView = (props: PartitionViewProps) => { }, [dimensions]); useEffect(() => { - if (!osmd || !soundPlayer) { + if (!osmd) { return; } if (props.timestamp > 0 && osmd.cursor.hidden && !osmd.cursor.iterator.EndReached) { @@ -138,7 +152,7 @@ const PartitionView = (props: PartitionViewProps) => { osmd.cursor.next(); if (osmd.cursor.iterator.EndReached) { osmd.cursor.hide(); // Lousy fix for https://github.com/opensheetmusicdisplay/opensheetmusicdisplay/issues/1338 - soundPlayer.stop(); + // soundPlayer.stop(); props.onEndReached(); } else { // Shamelessly stolen from https://github.com/jimutt/osmd-audio-player/blob/ec205a6e46ee50002c1fa8f5999389447bba7bbf/src/PlaybackEngine.ts#LL223C7-L224C1 diff --git a/front/components/PartitionVisualizer/PhaserCanvas.tsx b/front/components/PartitionVisualizer/PhaserCanvas.tsx new file mode 100644 index 0000000..1456c3f --- /dev/null +++ b/front/components/PartitionVisualizer/PhaserCanvas.tsx @@ -0,0 +1,101 @@ +// create a simple phaser effect with a canvas that can be easily imported as a react component + +import * as React from 'react'; +import { useEffect, useRef } from 'react'; +import Phaser from 'phaser'; +import { Asset, useAssets } from 'expo-asset'; +import { use } from 'matter'; + +const b64data = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAQAAAGKDAGaAAAFgUlEQVRYw7WXa2wUVRTH/20p7fZBW0p5iAplaUELCqEFlUCMYKwJKMYgaEwIUpQYNCIWRYgvQtTS6AeiKEqIQDBIAEFAEEm1DUVUoIqPVqhCC/IoammhC9vt/vywM7szu7NLLfHMl7n3nP/533vuPefMSHap8wNIoG57pxIYSErZDyCgB8qogTYUeKT+v2C+SlLanqA3v4EBwSdAasAI4CUUcA7wRsiBMbDIjQDQd4M0ogimtkrbMWSgAVzibUIouTIwfc4kJaVIkpT+PTxLiGplkHCo6Wq+uSOydlrYl3jD1xOSAQ5zdzcA1CJJK/ygOEk91kIhAMsQmeMDq9MRg7ojGExIINkrOAm8D4DHSg9QwiQApiDmeQzFWaDE2LUQ4DXjedimsAQaShgbqXj4NM4ISZpmVeTbdt6vGISrWtElfs76Fuig9+9hij7qFjbTs6GecEnxSdKa4DjbK0nFrzWFTI4yE/jVsvDsMaauP0JxX0ARviBgOg0A3GnZaVpO4N3llZRYYZrez8dhC/KRxrBVYSt37TPVgSh6GQN8FFxSQPLOqbsBSP3Wy2gLwJIRQcC91mPIOGyaTaTKARBxbllH7EtqZkJsgCRlvrTbF21JQiS3KN7pkFNH/BMOiJ+nq8vQTTNI8ChZnZabXJs7bxzf9zjAO/8kTLfNxynXwTpze6PlnG8/o16B+fzTfqCRgq1W62nXtYff1X1orjRiqzn2M8TgySlqAFjNuqDxFdIRKpWeag/Vo7zfJWXXei1ep3AaKDYDWypNvhLQnETk1+kWf/hC3iUrdGil0i3lAE8iRM/r1cpm3g0aNzAdgB8sACnvm8Bo0AuSLgLwKMeBQjosTEuCAJu0BSPgjsjo5WS0KyUM4Amqf6aO2cFRPUKU0cLAOhvgsgUAUM4u/PQz9lAGwC5/75eDgFlhAHuxKzNmynGtMwDtvMWOmIBahEjbYAACBWYCzY6A18kx3jI2WXolQJvRKMLqb6g5bbO0AnMPlbwSFdBrpwMA4FkOOwJ6mw08q/J8jChZnhtCR9HdfTo2IP6JyGQrfKbVGZBeETWdB723PwzQ/ZISr1IDchs9IcCtnasbWWMvCC3SNUhm2ophJ2raq9pyjycvjbju1yTje/604IzHllMXmH0256BGRBoPTipaO/jv4ktTPaNaC+rd02I5dqUsdTdUXiaGfOYbdKbPi2bnLkjLO3cszGJx+00fRLoemXlg1qkLVDOOxVyM6v4MM4kLlpQh6cObI22OIcatNh0nJj3X/8S2S3aTFhZwJ9/Z5raTZ71npZKUOWVxhPtDuBD5jZI7c+8DJ8/Giga7GcObzCMxMkFKJWlUYm6ztU0cYbihH7lccbv3cw9z+Tuq+yrG8gptNDGHBEcCSXGDjz7NDG6zWQyYLylhTyDSl1nK7VRYHLfxKmOpjCDcaH5dR5T1obPddekel9f95/BPs/ubwa+wB76acTzPRJ6hOWbYGnkEEb/wqlc96eu2CHAJ1cznLg5Fdf8lBQjxBqv87qbcx2MQJFd5HAjMyrSD0bxN6ABbWUiSQ9f4jQnNA7epjwNBSvWVGATmrX+M+xjjUGbLbHbtLLvU7w/dYf/0OrCV0ZTjjUEQ/WOxzJYh+QiRut5GkP6dz/jBms0kartA0ByWIekbbQQ9DnXYHG2gkIJOE7gc5jK32AgyavwOZ/A504N/AjG/px2entvtxf5Hoh5yByspZMN/JMjZbQ/RwRrf1W5RLZMY3Pkd7Ii8q5N71y9rae/CLbI/PY5qfKyk7ttvy13nj3aBIN6XslwZnW2TcX1KMlre8vk7RZB6QsVd7ccD3dUPXTwVhSCuI+lD80fi2iQhb1H+X5ssBEmn9KD+B7k54yut0XX/HfgvpUkmTvPggOsAAAAASUVORK5CYII='; + +const getPianoScene = (partitionB64: string) => { + class PianoScene extends Phaser.Scene { + async preload() { + // this.load.setBaseURL('http://labs.phaser.io'); + // this.load.setPath('content://assets/'); + // const imageData = await Asset.fromModule('./assets/raster-bw-64.png').downloadAsync(); + // this.load.image('raster', imageData.localUri); + } + + create() { + this.textures.addBase64('cursor', b64data); + this.textures.addBase64('raster', partitionB64); + + // wait for the image to be loaded, then create the sprites + this.textures.on('onload', () => { + const raster = this.add.image(300, 400, 'raster'); + const group = this.add.group(); + + group.createMultiple({ key: 'cursor', repeat: 8 }); + + let ci = 0; + const colors = [ + 0xef658c, 0xff9a52, 0xffdf00, 0x31ef8c, 0x21dfff, 0x31aade, 0x5275de, 0x9c55ad, + 0xbd208c, + ]; + + const _this = this; + + group.children.iterate((child) => { + child.x = 100; + child.y = 300; + child.depth = 9 - ci; + + child.tint = colors[ci]; + + ci++; + + _this.tweens.add({ + targets: child, + x: 900, + yoyo: true, + repeat: -1, + ease: 'Sine.easeInOut', + duration: 1500, + delay: 100 * ci, + }); + }); + }); + } + } + return PianoScene; +}; + +type PianoCursorPosition = { + // offset in pixels + x: number; + // timestamp in ms + timing: number; +}; + +type PhaserCanvasProps = { + partitionB64: string; + cursorPositions: PianoCursorPosition[]; +}; + +const PhaserCanvas = ({ partitionB64 }: PhaserCanvasProps) => { + const [game, setGame] = React.useState(null); + + useEffect(() => { + const PianoScene = getPianoScene(partitionB64); + + const config = { + type: Phaser.AUTO, + parent: 'phaser-canvas', + width: 1000, + height: 900, + scene: PianoScene, + scale: { + mode: Phaser.Scale.FIT, + autoCenter: Phaser.Scale.CENTER_BOTH, + }, + }; + + setGame(new Phaser.Game(config)); + }, []); + + return
; +}; + +export default PhaserCanvas; diff --git a/front/package.json b/front/package.json index f097628..380f499 100644 --- a/front/package.json +++ b/front/package.json @@ -50,6 +50,7 @@ "moti": "^0.22.0", "native-base": "^3.4.17", "opensheetmusicdisplay": "^1.7.5", + "phaser": "^3.60.0", "react": "18.1.0", "react-dom": "18.1.0", "react-i18next": "^11.18.3", diff --git a/front/views/HomeView.tsx b/front/views/HomeView.tsx index 131b74f..c4d2620 100644 --- a/front/views/HomeView.tsx +++ b/front/views/HomeView.tsx @@ -11,8 +11,14 @@ import Translate from '../components/Translate'; import TextButton from '../components/TextButton'; import Song from '../models/Song'; import { FontAwesome5 } from '@expo/vector-icons'; +import PhaserCanvas from '../components/PartitionVisualizer/PhaserCanvas'; + +const b64data = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAQAAAGKDAGaAAAFgUlEQVRYw7WXa2wUVRTH/20p7fZBW0p5iAplaUELCqEFlUCMYKwJKMYgaEwIUpQYNCIWRYgvQtTS6AeiKEqIQDBIAEFAEEm1DUVUoIqPVqhCC/IoammhC9vt/vywM7szu7NLLfHMl7n3nP/533vuPefMSHap8wNIoG57pxIYSErZDyCgB8qogTYUeKT+v2C+SlLanqA3v4EBwSdAasAI4CUUcA7wRsiBMbDIjQDQd4M0ogimtkrbMWSgAVzibUIouTIwfc4kJaVIkpT+PTxLiGplkHCo6Wq+uSOydlrYl3jD1xOSAQ5zdzcA1CJJK/ygOEk91kIhAMsQmeMDq9MRg7ojGExIINkrOAm8D4DHSg9QwiQApiDmeQzFWaDE2LUQ4DXjedimsAQaShgbqXj4NM4ISZpmVeTbdt6vGISrWtElfs76Fuig9+9hij7qFjbTs6GecEnxSdKa4DjbK0nFrzWFTI4yE/jVsvDsMaauP0JxX0ARviBgOg0A3GnZaVpO4N3llZRYYZrez8dhC/KRxrBVYSt37TPVgSh6GQN8FFxSQPLOqbsBSP3Wy2gLwJIRQcC91mPIOGyaTaTKARBxbllH7EtqZkJsgCRlvrTbF21JQiS3KN7pkFNH/BMOiJ+nq8vQTTNI8ChZnZabXJs7bxzf9zjAO/8kTLfNxynXwTpze6PlnG8/o16B+fzTfqCRgq1W62nXtYff1X1orjRiqzn2M8TgySlqAFjNuqDxFdIRKpWeag/Vo7zfJWXXei1ep3AaKDYDWypNvhLQnETk1+kWf/hC3iUrdGil0i3lAE8iRM/r1cpm3g0aNzAdgB8sACnvm8Bo0AuSLgLwKMeBQjosTEuCAJu0BSPgjsjo5WS0KyUM4Amqf6aO2cFRPUKU0cLAOhvgsgUAUM4u/PQz9lAGwC5/75eDgFlhAHuxKzNmynGtMwDtvMWOmIBahEjbYAACBWYCzY6A18kx3jI2WXolQJvRKMLqb6g5bbO0AnMPlbwSFdBrpwMA4FkOOwJ6mw08q/J8jChZnhtCR9HdfTo2IP6JyGQrfKbVGZBeETWdB723PwzQ/ZISr1IDchs9IcCtnasbWWMvCC3SNUhm2ophJ2raq9pyjycvjbju1yTje/604IzHllMXmH0256BGRBoPTipaO/jv4ktTPaNaC+rd02I5dqUsdTdUXiaGfOYbdKbPi2bnLkjLO3cszGJx+00fRLoemXlg1qkLVDOOxVyM6v4MM4kLlpQh6cObI22OIcatNh0nJj3X/8S2S3aTFhZwJ9/Z5raTZ71npZKUOWVxhPtDuBD5jZI7c+8DJ8/Giga7GcObzCMxMkFKJWlUYm6ztU0cYbihH7lccbv3cw9z+Tuq+yrG8gptNDGHBEcCSXGDjz7NDG6zWQyYLylhTyDSl1nK7VRYHLfxKmOpjCDcaH5dR5T1obPddekel9f95/BPs/ubwa+wB76acTzPRJ6hOWbYGnkEEb/wqlc96eu2CHAJ1cznLg5Fdf8lBQjxBqv87qbcx2MQJFd5HAjMyrSD0bxN6ABbWUiSQ9f4jQnNA7epjwNBSvWVGATmrX+M+xjjUGbLbHbtLLvU7w/dYf/0OrCV0ZTjjUEQ/WOxzJYh+QiRut5GkP6dz/jBms0kartA0ByWIekbbQQ9DnXYHG2gkIJOE7gc5jK32AgyavwOZ/A504N/AjG/px2entvtxf5Hoh5yByspZMN/JMjZbQ/RwRrf1W5RLZMY3Pkd7Ii8q5N71y9rae/CLbI/PY5qfKyk7ttvy13nj3aBIN6XslwZnW2TcX1KMlre8vk7RZB6QsVd7ccD3dUPXTwVhSCuI+lD80fi2iQhb1H+X5ssBEmn9KD+B7k54yut0XX/HfgvpUkmTvPggOsAAAAASUVORK5CYII='; + const HomeView = () => { + // return ; const navigation = useNavigation(); const userQuery = useQuery(API.getUserInfo); const playHistoryQuery = useQuery(API.getUserPlayHistory); diff --git a/front/views/PlayView.tsx b/front/views/PlayView.tsx index 652b61e..1920cf4 100644 --- a/front/views/PlayView.tsx +++ b/front/views/PlayView.tsx @@ -29,6 +29,7 @@ import { translate } from '../i18n/i18n'; import { ColorSchemeType } from 'native-base/lib/typescript/components/types'; import { useStopwatch } from 'react-use-precision-timer'; import PartitionView from '../components/PartitionView'; +import PartitionCoord from '../components/PartitionCoord'; import TextButton from '../components/TextButton'; import { MIDIAccess, MIDIMessageEvent, requestMIDIAccess } from '@motiz88/react-native-midi'; import * as Linking from 'expo-linking'; @@ -268,13 +269,21 @@ const PlayView = ({ songId, type, route }: RouteProps) => { - setPartitionRendered(true)} timestamp={Math.max(0, time)} onEndReached={() => { onEnd(); }} + /> */} + { + onEnd(); + }} + onPartitionReady={() => setPartitionRendered(true)} /> {!partitionRendered && } diff --git a/front/yarn.lock b/front/yarn.lock index 224b678..ae32b09 100644 --- a/front/yarn.lock +++ b/front/yarn.lock @@ -8921,6 +8921,11 @@ eventemitter3@^4.0.0: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== +eventemitter3@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4" + integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA== + events@^3.0.0, events@^3.2.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" @@ -14749,6 +14754,13 @@ pbkdf2@^3.0.3: safe-buffer "^5.0.1" sha.js "^2.4.8" +phaser@^3.60.0: + version "3.60.0" + resolved "https://registry.yarnpkg.com/phaser/-/phaser-3.60.0.tgz#8a555623e64c707482e6321485b4bda84604590d" + integrity sha512-IKUy35EnoEVcl2EmJ8WOyK4X8OoxHYdlhZLgRGpNrvD1fEagYffhVmwHcapE/tGiLgyrnezmXIo5RrH2NcrTHw== + dependencies: + eventemitter3 "^5.0.0" + picocolors@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f"