zoriya/Chromacase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

early Experiment working

Browse Source
This commit is contained in:
Clément Le Bihan
2023-07-20 00:08:20 +02:00
parent a3676fabf8
commit 125a7faf02
7 changed files with 198 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
front/components/PartitionCoord.tsx Normal file
View File

@@ -0,0 +1,40 @@
import * as React from 'react';
import PartitionView from './PartitionView';
import PhaserCanvas from './PartitionVisualizer/PhaserCanvas';
type PartitionCoordProps = {
// The Buffer of the MusicXML file retreived from the API
file: string;
onPartitionReady: () => void;
onEndReached: () => void;
// Timestamp of the play session, in milisecond
timestamp: number;
};
const PartitionCoord = ({
file,
onPartitionReady,
onEndReached,
timestamp,
}: PartitionCoordProps) => {
const [partitionB64, setPartitionB64] = React.useState<string | null>(null);
return (
<>
{!partitionB64 && (
<PartitionView
file={file}
onPartitionReady={(base64data) => {
setPartitionB64(base64data);
onPartitionReady();
}}
onEndReached={onEndReached}
timestamp={timestamp}
/>
)}
{partitionB64 && <PhaserCanvas partitionB64={partitionB64} cursorPositions={[]} />}
</>
);
};
export default PartitionCoord;

42
front/components/PartitionView.tsx
View File

@@ -17,7 +17,7 @@ import * as SAC from 'standardized-audio-context';
type PartitionViewProps = {
// The Buffer of the MusicXML file retreived from the API
file: string;
onPartitionReady: () => void;
onPartitionReady: (base64data: string) => void;
onEndReached: () => void;
// Timestamp of the play session, in milisecond
timestamp: number;
@@ -33,6 +33,7 @@ const PartitionView = (props: PartitionViewProps) => {
const OSMD_DIV_ID = 'osmd-div';
const options: IOSMDOptions = {
darkMode: colorScheme == 'dark',
backend: 'canvas',
drawComposer: false,
drawCredits: false,
drawLyrics: false,
@@ -68,14 +69,14 @@ const PartitionView = (props: PartitionViewProps) => {
// Put your hands together for https://github.com/jimutt/osmd-audio-player/blob/master/src/internals/noteHelpers.ts
const fixedKey =
note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0;
const midiNumber = note.halfTone - fixedKey * 12;
// console.log('Expecting midi ' + midiNumber);
const duration = getActualNoteLength(note);
const gain = note.ParentVoiceEntry.ParentVoice.Volume;
soundPlayer!.play(midiNumber.toString(), audioContext.currentTime, {
duration,
gain,
});
// const midiNumber = note.halfTone - fixedKey * 12;
// // console.log('Expecting midi ' + midiNumber);
// const duration = getActualNoteLength(note);
// const gain = note.ParentVoiceEntry.ParentVoice.Volume;
// soundPlayer!.play(midiNumber.toString(), audioContext.currentTime, {
// duration,
// gain,
// });
});
};
const getShortedNoteUnderCursor = () => {
@@ -88,16 +89,29 @@ const PartitionView = (props: PartitionViewProps) => {
useEffect(() => {
const _osmd = new OSMD(OSMD_DIV_ID, options);
Promise.all([
SoundFont.instrument(audioContext as unknown as AudioContext, 'electric_piano_1'),
// SoundFont.instrument(audioContext as unknown as AudioContext, 'electric_piano_1'),
_osmd.load(props.file),
]).then(([player]) => {
setSoundPlayer(player);
// setSoundPlayer(player);
_osmd.render();
_osmd.cursor.show();
// get the current cursor position
const curPos = [];
while (!_osmd.cursor.iterator.EndReached) {
curPos.push(_osmd.cursor.cursorElement.offsetLeft);
_osmd.cursor.next();
}
console.log('curPos', curPos);
_osmd.cursor.reset();
_osmd.cursor.hide();
console.log("timestamp cursor", _osmd.cursor.iterator.CurrentSourceTimestamp);
console.log("timestamp cursor", _osmd.cursor.iterator.CurrentVoiceEntries);
console.log("current measure index", _osmd.cursor.iterator.CurrentMeasureIndex);
const osmdCanvas = document.querySelector( "#" + OSMD_DIV_ID + " canvas");
// Ty https://github.com/jimutt/osmd-audio-player/blob/ec205a6e46ee50002c1fa8f5999389447bba7bbf/src/PlaybackEngine.ts#LL77C12-L77C63
const bpm = _osmd.Sheet.HasBPMInfo ? _osmd.Sheet.getExpressionsStartTempoInBPM() : 60;
setWholeNoteLength(Math.round((60 / bpm) * 4000));
props.onPartitionReady();
props.onPartitionReady(osmdCanvas.toDataURL());
// Do not show cursor before actuall start
});
setOsmd(_osmd);
@@ -114,7 +128,7 @@ const PartitionView = (props: PartitionViewProps) => {
}, [dimensions]);
useEffect(() => {
if (!osmd || !soundPlayer) {
if (!osmd) {
return;
}
if (props.timestamp > 0 && osmd.cursor.hidden && !osmd.cursor.iterator.EndReached) {
@@ -138,7 +152,7 @@ const PartitionView = (props: PartitionViewProps) => {
osmd.cursor.next();
if (osmd.cursor.iterator.EndReached) {
osmd.cursor.hide(); // Lousy fix for https://github.com/opensheetmusicdisplay/opensheetmusicdisplay/issues/1338
soundPlayer.stop();
// soundPlayer.stop();
props.onEndReached();
} else {
// Shamelessly stolen from https://github.com/jimutt/osmd-audio-player/blob/ec205a6e46ee50002c1fa8f5999389447bba7bbf/src/PlaybackEngine.ts#LL223C7-L224C1

101
front/components/PartitionVisualizer/PhaserCanvas.tsx Normal file
View File

@@ -0,0 +1,101 @@
// create a simple phaser effect with a canvas that can be easily imported as a react component
import * as React from 'react';
import { useEffect, useRef } from 'react';
import Phaser from 'phaser';
import { Asset, useAssets } from 'expo-asset';
import { use } from 'matter';
const b64data =
'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAQAAAGKDAGaAAAFgUlEQVRYw7WXa2wUVRTH/20p7fZBW0p5iAplaUELCqEFlUCMYKwJKMYgaEwIUpQYNCIWRYgvQtTS6AeiKEqIQDBIAEFAEEm1DUVUoIqPVqhCC/IoammhC9vt/vywM7szu7NLLfHMl7n3nP/533vuPefMSHap8wNIoG57pxIYSErZDyCgB8qogTYUeKT+v2C+SlLanqA3v4EBwSdAasAI4CUUcA7wRsiBMbDIjQDQd4M0ogimtkrbMWSgAVzibUIouTIwfc4kJaVIkpT+PTxLiGplkHCo6Wq+uSOydlrYl3jD1xOSAQ5zdzcA1CJJK/ygOEk91kIhAMsQmeMDq9MRg7ojGExIINkrOAm8D4DHSg9QwiQApiDmeQzFWaDE2LUQ4DXjedimsAQaShgbqXj4NM4ISZpmVeTbdt6vGISrWtElfs76Fuig9+9hij7qFjbTs6GecEnxSdKa4DjbK0nFrzWFTI4yE/jVsvDsMaauP0JxX0ARviBgOg0A3GnZaVpO4N3llZRYYZrez8dhC/KRxrBVYSt37TPVgSh6GQN8FFxSQPLOqbsBSP3Wy2gLwJIRQcC91mPIOGyaTaTKARBxbllH7EtqZkJsgCRlvrTbF21JQiS3KN7pkFNH/BMOiJ+nq8vQTTNI8ChZnZabXJs7bxzf9zjAO/8kTLfNxynXwTpze6PlnG8/o16B+fzTfqCRgq1W62nXtYff1X1orjRiqzn2M8TgySlqAFjNuqDxFdIRKpWeag/Vo7zfJWXXei1ep3AaKDYDWypNvhLQnETk1+kWf/hC3iUrdGil0i3lAE8iRM/r1cpm3g0aNzAdgB8sACnvm8Bo0AuSLgLwKMeBQjosTEuCAJu0BSPgjsjo5WS0KyUM4Amqf6aO2cFRPUKU0cLAOhvgsgUAUM4u/PQz9lAGwC5/75eDgFlhAHuxKzNmynGtMwDtvMWOmIBahEjbYAACBWYCzY6A18kx3jI2WXolQJvRKMLqb6g5bbO0AnMPlbwSFdBrpwMA4FkOOwJ6mw08q/J8jChZnhtCR9HdfTo2IP6JyGQrfKbVGZBeETWdB723PwzQ/ZISr1IDchs9IcCtnasbWWMvCC3SNUhm2ophJ2raq9pyjycvjbju1yTje/604IzHllMXmH0256BGRBoPTipaO/jv4ktTPaNaC+rd02I5dqUsdTdUXiaGfOYbdKbPi2bnLkjLO3cszGJx+00fRLoemXlg1qkLVDOOxVyM6v4MM4kLlpQh6cObI22OIcatNh0nJj3X/8S2S3aTFhZwJ9/Z5raTZ71npZKUOWVxhPtDuBD5jZI7c+8DJ8/Giga7GcObzCMxMkFKJWlUYm6ztU0cYbihH7lccbv3cw9z+Tuq+yrG8gptNDGHBEcCSXGDjz7NDG6zWQyYLylhTyDSl1nK7VRYHLfxKmOpjCDcaH5dR5T1obPddekel9f95/BPs/ubwa+wB76acTzPRJ6hOWbYGnkEEb/wqlc96eu2CHAJ1cznLg5Fdf8lBQjxBqv87qbcx2MQJFd5HAjMyrSD0bxN6ABbWUiSQ9f4jQnNA7epjwNBSvWVGATmrX+M+xjjUGbLbHbtLLvU7w/dYf/0OrCV0ZTjjUEQ/WOxzJYh+QiRut5GkP6dz/jBms0kartA0ByWIekbbQQ9DnXYHG2gkIJOE7gc5jK32AgyavwOZ/A504N/AjG/px2entvtxf5Hoh5yByspZMN/JMjZbQ/RwRrf1W5RLZMY3Pkd7Ii8q5N71y9rae/CLbI/PY5qfKyk7ttvy13nj3aBIN6XslwZnW2TcX1KMlre8vk7RZB6QsVd7ccD3dUPXTwVhSCuI+lD80fi2iQhb1H+X5ssBEmn9KD+B7k54yut0XX/HfgvpUkmTvPggOsAAAAASUVORK5CYII=';
const getPianoScene = (partitionB64: string) => {
class PianoScene extends Phaser.Scene {
async preload() {
// this.load.setBaseURL('http://labs.phaser.io');
// this.load.setPath('content://assets/');
// const imageData = await Asset.fromModule('./assets/raster-bw-64.png').downloadAsync();
// this.load.image('raster', imageData.localUri);
}
create() {
this.textures.addBase64('cursor', b64data);
this.textures.addBase64('raster', partitionB64);
// wait for the image to be loaded, then create the sprites
this.textures.on('onload', () => {
const raster = this.add.image(300, 400, 'raster');
const group = this.add.group();
group.createMultiple({ key: 'cursor', repeat: 8 });
let ci = 0;
const colors = [
0xef658c, 0xff9a52, 0xffdf00, 0x31ef8c, 0x21dfff, 0x31aade, 0x5275de, 0x9c55ad,
0xbd208c,
];
const _this = this;
group.children.iterate((child) => {
child.x = 100;
child.y = 300;
child.depth = 9 - ci;
child.tint = colors[ci];
ci++;
_this.tweens.add({
targets: child,
x: 900,
yoyo: true,
repeat: -1,
ease: 'Sine.easeInOut',
duration: 1500,
delay: 100 * ci,
});
});
});
}
}
return PianoScene;
};
type PianoCursorPosition = {
// offset in pixels
x: number;
// timestamp in ms
timing: number;
};
type PhaserCanvasProps = {
partitionB64: string;
cursorPositions: PianoCursorPosition[];
};
const PhaserCanvas = ({ partitionB64 }: PhaserCanvasProps) => {
const [game, setGame] = React.useState<Phaser.Game | null>(null);
useEffect(() => {
const PianoScene = getPianoScene(partitionB64);
const config = {
type: Phaser.AUTO,
parent: 'phaser-canvas',
width: 1000,
height: 900,
scene: PianoScene,
scale: {
mode: Phaser.Scale.FIT,
autoCenter: Phaser.Scale.CENTER_BOTH,
},
};
setGame(new Phaser.Game(config));
}, []);
return <div id="phaser-canvas"></div>;
};
export default PhaserCanvas;

1
front/package.json
View File

@@ -50,6 +50,7 @@
"moti": "^0.22.0",
"native-base": "^3.4.17",
"opensheetmusicdisplay": "^1.7.5",
"phaser": "^3.60.0",
"react": "18.1.0",
"react-dom": "18.1.0",
"react-i18next": "^11.18.3",

6
front/views/HomeView.tsx
View File

@@ -11,8 +11,14 @@ import Translate from '../components/Translate';
import TextButton from '../components/TextButton';
import Song from '../models/Song';
import { FontAwesome5 } from '@expo/vector-icons';
import PhaserCanvas from '../components/PartitionVisualizer/PhaserCanvas';
const b64data =
'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAQAAAGKDAGaAAAFgUlEQVRYw7WXa2wUVRTH/20p7fZBW0p5iAplaUELCqEFlUCMYKwJKMYgaEwIUpQYNCIWRYgvQtTS6AeiKEqIQDBIAEFAEEm1DUVUoIqPVqhCC/IoammhC9vt/vywM7szu7NLLfHMl7n3nP/533vuPefMSHap8wNIoG57pxIYSErZDyCgB8qogTYUeKT+v2C+SlLanqA3v4EBwSdAasAI4CUUcA7wRsiBMbDIjQDQd4M0ogimtkrbMWSgAVzibUIouTIwfc4kJaVIkpT+PTxLiGplkHCo6Wq+uSOydlrYl3jD1xOSAQ5zdzcA1CJJK/ygOEk91kIhAMsQmeMDq9MRg7ojGExIINkrOAm8D4DHSg9QwiQApiDmeQzFWaDE2LUQ4DXjedimsAQaShgbqXj4NM4ISZpmVeTbdt6vGISrWtElfs76Fuig9+9hij7qFjbTs6GecEnxSdKa4DjbK0nFrzWFTI4yE/jVsvDsMaauP0JxX0ARviBgOg0A3GnZaVpO4N3llZRYYZrez8dhC/KRxrBVYSt37TPVgSh6GQN8FFxSQPLOqbsBSP3Wy2gLwJIRQcC91mPIOGyaTaTKARBxbllH7EtqZkJsgCRlvrTbF21JQiS3KN7pkFNH/BMOiJ+nq8vQTTNI8ChZnZabXJs7bxzf9zjAO/8kTLfNxynXwTpze6PlnG8/o16B+fzTfqCRgq1W62nXtYff1X1orjRiqzn2M8TgySlqAFjNuqDxFdIRKpWeag/Vo7zfJWXXei1ep3AaKDYDWypNvhLQnETk1+kWf/hC3iUrdGil0i3lAE8iRM/r1cpm3g0aNzAdgB8sACnvm8Bo0AuSLgLwKMeBQjosTEuCAJu0BSPgjsjo5WS0KyUM4Amqf6aO2cFRPUKU0cLAOhvgsgUAUM4u/PQz9lAGwC5/75eDgFlhAHuxKzNmynGtMwDtvMWOmIBahEjbYAACBWYCzY6A18kx3jI2WXolQJvRKMLqb6g5bbO0AnMPlbwSFdBrpwMA4FkOOwJ6mw08q/J8jChZnhtCR9HdfTo2IP6JyGQrfKbVGZBeETWdB723PwzQ/ZISr1IDchs9IcCtnasbWWMvCC3SNUhm2ophJ2raq9pyjycvjbju1yTje/604IzHllMXmH0256BGRBoPTipaO/jv4ktTPaNaC+rd02I5dqUsdTdUXiaGfOYbdKbPi2bnLkjLO3cszGJx+00fRLoemXlg1qkLVDOOxVyM6v4MM4kLlpQh6cObI22OIcatNh0nJj3X/8S2S3aTFhZwJ9/Z5raTZ71npZKUOWVxhPtDuBD5jZI7c+8DJ8/Giga7GcObzCMxMkFKJWlUYm6ztU0cYbihH7lccbv3cw9z+Tuq+yrG8gptNDGHBEcCSXGDjz7NDG6zWQyYLylhTyDSl1nK7VRYHLfxKmOpjCDcaH5dR5T1obPddekel9f95/BPs/ubwa+wB76acTzPRJ6hOWbYGnkEEb/wqlc96eu2CHAJ1cznLg5Fdf8lBQjxBqv87qbcx2MQJFd5HAjMyrSD0bxN6ABbWUiSQ9f4jQnNA7epjwNBSvWVGATmrX+M+xjjUGbLbHbtLLvU7w/dYf/0OrCV0ZTjjUEQ/WOxzJYh+QiRut5GkP6dz/jBms0kartA0ByWIekbbQQ9DnXYHG2gkIJOE7gc5jK32AgyavwOZ/A504N/AjG/px2entvtxf5Hoh5yByspZMN/JMjZbQ/RwRrf1W5RLZMY3Pkd7Ii8q5N71y9rae/CLbI/PY5qfKyk7ttvy13nj3aBIN6XslwZnW2TcX1KMlre8vk7RZB6QsVd7ccD3dUPXTwVhSCuI+lD80fi2iQhb1H+X5ssBEmn9KD+B7k54yut0XX/HfgvpUkmTvPggOsAAAAASUVORK5CYII=';
const HomeView = () => {
// return <PhaserCanvas partitionB64={b64data} />;
const navigation = useNavigation();
const userQuery = useQuery(API.getUserInfo);
const playHistoryQuery = useQuery(API.getUserPlayHistory);

11
front/views/PlayView.tsx
View File

@@ -29,6 +29,7 @@ import { translate } from '../i18n/i18n';
import { ColorSchemeType } from 'native-base/lib/typescript/components/types';
import { useStopwatch } from 'react-use-precision-timer';
import PartitionView from '../components/PartitionView';
import PartitionCoord from '../components/PartitionCoord';
import TextButton from '../components/TextButton';
import { MIDIAccess, MIDIMessageEvent, requestMIDIAccess } from '@motiz88/react-native-midi';
import * as Linking from 'expo-linking';
@@ -268,13 +269,21 @@ const PlayView = ({ songId, type, route }: RouteProps<PlayViewProps>) => {
</Animated.View>
</HStack>
<View style={{ flexGrow: 1, justifyContent: 'center' }}>
<PartitionView
{/* <PartitionView
file={musixml.data}
onPartitionReady={() => setPartitionRendered(true)}
timestamp={Math.max(0, time)}
onEndReached={() => {
onEnd();
}}
/> */}
<PartitionCoord
file={musixml.data}
timestamp={Math.max(0, time)}
onEndReached={() => {
onEnd();
}}
onPartitionReady={() => setPartitionRendered(true)}
/>
{!partitionRendered && <LoadingComponent />}
</View>

12
front/yarn.lock
View File

@@ -8921,6 +8921,11 @@ eventemitter3@^4.0.0:
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
eventemitter3@^5.0.0:
version "5.0.1"
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4"
integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==
events@^3.0.0, events@^3.2.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
@@ -14749,6 +14754,13 @@ pbkdf2@^3.0.3:
safe-buffer "^5.0.1"
sha.js "^2.4.8"
phaser@^3.60.0:
version "3.60.0"
resolved "https://registry.yarnpkg.com/phaser/-/phaser-3.60.0.tgz#8a555623e64c707482e6321485b4bda84604590d"
integrity sha512-IKUy35EnoEVcl2EmJ8WOyK4X8OoxHYdlhZLgRGpNrvD1fEagYffhVmwHcapE/tGiLgyrnezmXIo5RrH2NcrTHw==
dependencies:
eventemitter3 "^5.0.0"
picocolors@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f"