From 30fcacbec6eecf9d07eb14a033499d75ff5ac5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Le=20Bihan?= Date: Mon, 4 Sep 2023 15:23:52 +0200 Subject: [PATCH] Now using redux to not create sound player every time the phaser is also implicitely cached --- .../PartitionVisualizer/PhaserCanvas.tsx | 117 ++++++++----- front/models/SearchHistory.ts | 4 +- front/package.json | 2 + front/state/SoundPlayerSlice.ts | 19 +++ front/state/Store.ts | 6 +- front/views/PlayView.tsx | 156 +++++++++--------- front/yarn.lock | 42 +++++ 7 files changed, 220 insertions(+), 126 deletions(-) create mode 100644 front/state/SoundPlayerSlice.ts diff --git a/front/components/PartitionVisualizer/PhaserCanvas.tsx b/front/components/PartitionVisualizer/PhaserCanvas.tsx index 51fbbf7..ea0e6b4 100644 --- a/front/components/PartitionVisualizer/PhaserCanvas.tsx +++ b/front/components/PartitionVisualizer/PhaserCanvas.tsx @@ -5,23 +5,28 @@ import { useEffect, useRef } from 'react'; import Phaser from 'phaser'; import useColorScheme from '../../hooks/colorScheme'; import { PartitionContext } from '../../views/PlayView'; -import { on } from 'events'; +import store, { RootState, useSelector } from '../../state/Store'; +import { setSoundPlayer as setSPStore } from '../../state/SoundPlayerSlice'; +import { useDispatch } from 'react-redux'; import SoundFont from 'soundfont-player'; import * as SAC from 'standardized-audio-context'; +import { SplendidGrandPiano, CacheStorage } from 'smplr'; +// import * as Tone from 'tone'; let globalTimestamp = 0; let globalStatus: 'playing' | 'paused' | 'stopped' = 'playing'; -const playNotes = (notes: any[], soundPlayer: SoundFont.Player, audioContext: SAC.AudioContext) => { +const isValidSoundPlayer = (soundPlayer: SplendidGrandPiano | undefined) => { + return soundPlayer && soundPlayer.loaded; +} + +const playNotes = (notes: any[], soundPlayer: SplendidGrandPiano) => { notes.forEach(({ note, duration }) => { const fixedKey = note.ParentVoiceEntry.ParentVoice.Parent.SubInstruments.at(0)?.fixedKey ?? 0; const midiNumber = note.halfTone - fixedKey * 12; const gain = note.ParentVoiceEntry.ParentVoice.Volume; - soundPlayer!.play(midiNumber.toString(), audioContext.currentTime, { - duration, - gain, - }); + soundPlayer.start({ note: midiNumber, duration, velocity: gain * 127 }); }); }; @@ -29,8 +34,7 @@ const getPianoScene = ( partitionB64: string, cursorPositions: PianoCursorPosition[], onEndReached: () => void, - soundPlayer: SoundFont.Player, - audioContext: SAC.AudioContext, + soundPlayer: SplendidGrandPiano, colorScheme: 'light' | 'dark' ) => { class PianoScene extends Phaser.Scene { @@ -67,7 +71,7 @@ const getPianoScene = ( return false; }); if (cP) { - playNotes(cP.notes, soundPlayer, audioContext); + playNotes(cP.notes, soundPlayer); const tw = { targets: this!.cursor, x: cP!.x, @@ -76,7 +80,6 @@ const getPianoScene = ( }; if (this.cursorPositionsIdx === cursorPositions.length - 1) { tw.onComplete = () => { - soundPlayer.stop(); onEndReached(); }; } @@ -88,6 +91,21 @@ const getPianoScene = ( return PianoScene; }; +const getSoundPlayer = async (audioContext: AudioContext) => { + const soundPlayerStore = store.getState().soundPlayer.soundPlayer; + if (soundPlayerStore) { + console.log('csp', soundPlayerStore); + return soundPlayerStore as unknown as SplendidGrandPiano; + } + const soundPlayer = await new SplendidGrandPiano(audioContext, { + storage: new CacheStorage(), + }).loaded(); + console.log('sp', soundPlayer); + setSPStore(soundPlayer); + console.log('asp', soundPlayer); + return soundPlayer; +}; + export type PianoCursorPosition = { // offset in pixels x: number; @@ -110,43 +128,62 @@ export type PhaserCanvasProps = { const PhaserCanvas = ({ partitionB64, cursorPositions, onEndReached }: PhaserCanvasProps) => { const colorScheme = useColorScheme(); - const audioContext = new SAC.AudioContext(); - const [soundPlayer, setSoundPlayer] = React.useState(); + const dispatch = useDispatch(); + const soundPlayer = useSelector((state: RootState) => state.soundPlayer.soundPlayer); const { timestamp } = React.useContext(PartitionContext); const [game, setGame] = React.useState(null); globalTimestamp = timestamp; + useEffect(() => { - Promise.resolve( - SoundFont.instrument(audioContext as unknown as AudioContext, 'electric_piano_1') - ).then((sound) => { - setSoundPlayer(sound); - - const pianoScene = getPianoScene( - partitionB64, - cursorPositions, - onEndReached, - sound, - audioContext, - colorScheme - ); - - const config = { - type: Phaser.AUTO, - parent: 'phaser-canvas', - width: 1000, - height: 400, - scene: pianoScene, - scale: { - mode: Phaser.Scale.FIT, - autoCenter: Phaser.Scale.CENTER_BOTH, - }, - }; - - setGame(new Phaser.Game(config)); - }); + if (isValidSoundPlayer(soundPlayer)) { + console.log('cache soundplayer', soundPlayer); + return; + } + console.log('creating soundplayer'); + new SplendidGrandPiano(new AudioContext(), { + storage: new CacheStorage(), + }) + .loaded() + .then((sp) => { + console.log('sp', sp); + dispatch(setSPStore(sp)); + }); }, []); + useEffect(() => { + console.log('soundPlayer', soundPlayer); + if (!isValidSoundPlayer(soundPlayer) || !soundPlayer) return; + const pianoScene = getPianoScene( + partitionB64, + cursorPositions, + onEndReached, + soundPlayer, + colorScheme + ); + + const config = { + type: Phaser.AUTO, + parent: 'phaser-canvas', + width: 1000, + height: 400, + scene: pianoScene, + scale: { + mode: Phaser.Scale.FIT, + autoCenter: Phaser.Scale.CENTER_BOTH, + }, + }; + + setGame(new Phaser.Game(config)); + return () => { + console.log('destroying phaser game sp'); + if (game) { + // currently the condition is always false + game.destroy(true); + } + } + }, [soundPlayer]); + return
; }; diff --git a/front/models/SearchHistory.ts b/front/models/SearchHistory.ts index 05c433b..a7a995c 100644 --- a/front/models/SearchHistory.ts +++ b/front/models/SearchHistory.ts @@ -2,7 +2,7 @@ import Model, { ModelValidator } from './Model'; import * as yup from 'yup'; import ResponseHandler from './ResponseHandler'; -export const SearchType = ['song', 'artist', 'album'] as const; +export const SearchType = ['song', 'artist', 'album', 'genre'] as const; export type SearchType = (typeof SearchType)[number]; const SearchHistoryValidator = yup @@ -27,7 +27,7 @@ export const SearchHistoryHandler: ResponseHandler< interface SearchHistory extends Model { query: string; - type: 'song' | 'artist' | 'album' | 'genre'; + type: SearchType; userId: number; timestamp: Date; } diff --git a/front/package.json b/front/package.json index 380f499..de6499c 100644 --- a/front/package.json +++ b/front/package.json @@ -69,8 +69,10 @@ "react-timer-hook": "^3.0.5", "react-use-precision-timer": "^3.3.1", "redux-persist": "^6.0.0", + "smplr": "^0.6.1", "soundfont-player": "^0.12.0", "standardized-audio-context": "^25.3.51", + "tone": "^14.7.77", "type-fest": "^3.6.0", "yup": "^1.2.0" }, diff --git a/front/state/SoundPlayerSlice.ts b/front/state/SoundPlayerSlice.ts new file mode 100644 index 0000000..c45fc9f --- /dev/null +++ b/front/state/SoundPlayerSlice.ts @@ -0,0 +1,19 @@ +import { createSlice, PayloadAction } from '@reduxjs/toolkit'; +import { SplendidGrandPiano } from 'smplr'; + +export const soundPlayerSlice = createSlice({ + name: 'soundPlayer', + initialState: { + soundPlayer: undefined as SplendidGrandPiano | undefined, + }, + reducers: { + setSoundPlayer: (state, action: PayloadAction) => { + state.soundPlayer = action.payload; + }, + unsetSoundPlayer: (state) => { + state.soundPlayer = undefined; + }, + }, +}); +export const { setSoundPlayer, unsetSoundPlayer } = soundPlayerSlice.actions; +export default soundPlayerSlice.reducer; diff --git a/front/state/Store.ts b/front/state/Store.ts index a113422..853871b 100644 --- a/front/state/Store.ts +++ b/front/state/Store.ts @@ -1,5 +1,6 @@ import userReducer from '../state/UserSlice'; -import settingsReduder from './SettingsSlice'; +import settingsReducer from './SettingsSlice'; +import SoundPlayerSliceReducer from './SoundPlayerSlice'; import { StateFromReducersMapObject, configureStore } from '@reduxjs/toolkit'; import languageReducer from './LanguageSlice'; import { @@ -28,7 +29,8 @@ const persistConfig = { const reducers = { user: userReducer, language: languageReducer, - settings: settingsReduder, + settings: settingsReducer, + soundPlayer: SoundPlayerSliceReducer, }; type State = StateFromReducersMapObject; diff --git a/front/views/PlayView.tsx b/front/views/PlayView.tsx index 63e7afc..d54890e 100644 --- a/front/views/PlayView.tsx +++ b/front/views/PlayView.tsx @@ -99,15 +99,13 @@ const PlayView = ({ songId, type, route }: RouteProps) => { const onPause = () => { stopwatch.pause(); setPause(true); - if (webSocket.current?.readyState == WebSocket.OPEN) { - webSocket.current?.send( - JSON.stringify({ - type: 'pause', - paused: true, - time: getElapsedTime(), - }) - ); - } + webSocket.current?.send( + JSON.stringify({ + type: 'pause', + paused: true, + time: getElapsedTime(), + }) + ); }; const onResume = () => { if (stopwatch.isStarted()) { @@ -116,26 +114,20 @@ const PlayView = ({ songId, type, route }: RouteProps) => { stopwatch.start(); } setPause(false); - if (webSocket.current?.readyState == WebSocket.OPEN) { - webSocket.current?.send( - JSON.stringify({ - type: 'pause', - paused: false, - time: getElapsedTime(), - }) - ); - } + webSocket.current?.send( + JSON.stringify({ + type: 'pause', + paused: false, + time: getElapsedTime(), + }) + ); }; const onEnd = () => { - setTime(0); - stopwatch.stop(); - if (webSocket.current?.readyState == WebSocket.OPEN) { - webSocket.current?.send( - JSON.stringify({ - type: 'end', - }) - ); - } + // webSocket.current?.send( + // JSON.stringify({ + // type: 'end', + // }) + // ); }; const onMIDISuccess = (access: MIDIAccess) => { @@ -147,63 +139,63 @@ const PlayView = ({ songId, type, route }: RouteProps) => { } setMidiKeyboardFound(true); let inputIndex = 0; - webSocket.current = new WebSocket(scoroBaseApiUrl); - webSocket.current.onopen = () => { - webSocket.current!.send( - JSON.stringify({ - type: 'start', - id: song.data!.id, - mode: type, - bearer: accessToken, - }) - ); - }; - webSocket.current.onmessage = (message) => { - try { - const data = JSON.parse(message.data); - if (data.type == 'end') { - navigation.navigate('Score', { songId: song.data!.id, ...data }); - return; - } - const points = data.info.score; - const maxPoints = data.info.max_score || 1; + //webSocket.current = new WebSocket(scoroBaseApiUrl); + // webSocket.current.onopen = () => { + // webSocket.current!.send( + // JSON.stringify({ + // type: 'start', + // id: song.data!.id, + // mode: type, + // bearer: accessToken, + // }) + // ); + // }; + // webSocket.current.onmessage = (message) => { + // try { + // const data = JSON.parse(message.data); + // if (data.type == 'end') { + // navigation.navigate('Score', { songId: song.data!.id, ...data }); + // return; + // } + // const points = data.info.score; + // const maxPoints = data.info.max_score || 1; - setScore(Math.floor((Math.max(points, 0) * 100) / maxPoints)); + // setScore(Math.floor((Math.max(points, 0) * 100) / maxPoints)); - let formattedMessage = ''; - let messageColor: ColorSchemeType | undefined; + // let formattedMessage = ''; + // let messageColor: ColorSchemeType | undefined; - if (data.type == 'miss') { - formattedMessage = translate('missed'); - messageColor = 'black'; - } else if (data.type == 'timing' || data.type == 'duration') { - formattedMessage = translate(data[data.type]); - switch (data[data.type]) { - case 'perfect': - messageColor = 'green'; - break; - case 'great': - messageColor = 'blue'; - break; - case 'short': - case 'long': - case 'good': - messageColor = 'lightBlue'; - break; - case 'too short': - case 'too long': - case 'wrong': - messageColor = 'trueGray'; - break; - default: - break; - } - } - setLastScoreMessage({ content: formattedMessage, color: messageColor }); - } catch (e) { - console.error(e); - } - }; + // if (data.type == 'miss') { + // formattedMessage = translate('missed'); + // messageColor = 'black'; + // } else if (data.type == 'timing' || data.type == 'duration') { + // formattedMessage = translate(data[data.type]); + // switch (data[data.type]) { + // case 'perfect': + // messageColor = 'green'; + // break; + // case 'great': + // messageColor = 'blue'; + // break; + // case 'short': + // case 'long': + // case 'good': + // messageColor = 'lightBlue'; + // break; + // case 'too short': + // case 'too long': + // case 'wrong': + // messageColor = 'trueGray'; + // break; + // default: + // break; + // } + // } + // setLastScoreMessage({ content: formattedMessage, color: messageColor }); + // } catch (e) { + // console.error(e); + // } + // }; inputs.forEach((input) => { if (inputIndex != 0) { return; @@ -232,7 +224,7 @@ const PlayView = ({ songId, type, route }: RouteProps) => { ScreenOrientation.lockAsync(ScreenOrientation.OrientationLock.LANDSCAPE).catch(() => {}); const interval = setInterval(() => { setTime(() => getElapsedTime()); // Countdown - }, 1); + }, 10); return () => { ScreenOrientation.unlockAsync().catch(() => {}); diff --git a/front/yarn.lock b/front/yarn.lock index ae32b09..e95f432 100644 --- a/front/yarn.lock +++ b/front/yarn.lock @@ -1246,6 +1246,13 @@ dependencies: regenerator-runtime "^0.13.11" +"@babel/runtime@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.6.tgz#57d64b9ae3cff1d67eb067ae117dac087f5bd438" + integrity sha512-wDb5pWm4WDdF6LFUde3Jl8WzPA+3ZbxYqkC6xAXuD3irdEHN1k0NfTRrJD8ZD378SJ61miMLCqIOXYhd8x+AJQ== + dependencies: + regenerator-runtime "^0.13.11" + "@babel/runtime@~7.5.4": version "7.5.5" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.5.5.tgz#74fba56d35efbeca444091c7850ccd494fd2f132" @@ -6027,6 +6034,14 @@ automation-events@^6.0.4: "@babel/runtime" "^7.22.3" tslib "^2.5.3" +automation-events@^6.0.8: + version "6.0.8" + resolved "https://registry.yarnpkg.com/automation-events/-/automation-events-6.0.8.tgz#52929699924cd791eaefc51916ffc033c4d0f42f" + integrity sha512-OXI9rEbA0LwWr+Tmvka4EHtVHBIVw8KD2NM7fIGjd4dyGnuiM3ULZL+Jlo4aKXZDY98raT4R4rEDOHAbz8Jm9A== + dependencies: + "@babel/runtime" "^7.22.6" + tslib "^2.6.1" + autoprefixer@^9.8.6: version "9.8.8" resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.8.tgz#fd4bd4595385fa6f06599de749a4d5f7a474957a" @@ -16987,6 +17002,11 @@ smart-buffer@^4.2.0: resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== +smplr@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/smplr/-/smplr-0.6.1.tgz#f24cbe7ce3ad318bb6ce226d9aa933d1cab7dc56" + integrity sha512-040QDtYRavqIje9346zWBYDc3oN/ARSZmheOGELAujQVYr3p4e8nrOsojH3VQsE0zcrAhjJ4MDeg74qIHQCC7A== + snapdragon-node@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" @@ -17266,6 +17286,15 @@ stacktrace-parser@^0.1.3: dependencies: type-fest "^0.7.1" +standardized-audio-context@^25.1.8: + version "25.3.55" + resolved "https://registry.yarnpkg.com/standardized-audio-context/-/standardized-audio-context-25.3.55.tgz#4d87ea6052de80ecf5abf56eb71ecd71f7e52e4e" + integrity sha512-ym9g7FZ5S1FykbQ1///ktTJgk+zTtGF1hGR/BFRQjRkN6G2Xy9GbL5kOcM7DlzflV2yJtqVwfU2gL042b1oHwg== + dependencies: + "@babel/runtime" "^7.22.6" + automation-events "^6.0.8" + tslib "^2.6.1" + standardized-audio-context@^25.3.51: version "25.3.51" resolved "https://registry.yarnpkg.com/standardized-audio-context/-/standardized-audio-context-25.3.51.tgz#0eb54629355d1ddf2070897e586eaa8dfec8c0f5" @@ -18038,6 +18067,14 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tone@^14.7.77: + version "14.7.77" + resolved "https://registry.yarnpkg.com/tone/-/tone-14.7.77.tgz#12a2a9f033952ccdb552275a6384ca5d36d4b5ed" + integrity sha512-tCfK73IkLHyzoKUvGq47gyDyxiKLFvKiVCOobynGgBB9Dl0NkxTM2p+eRJXyCYrjJwy9Y0XCMqD3uOYsYt2Fdg== + dependencies: + standardized-audio-context "^25.1.8" + tslib "^2.0.1" + toposort@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/toposort/-/toposort-2.0.2.tgz#ae21768175d1559d48bef35420b2f4962f09c330" @@ -18125,6 +18162,11 @@ tslib@^2.5.3: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.3.tgz#24944ba2d990940e6e982c4bea147aba80209913" integrity sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w== +tslib@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.1.tgz#fd8c9a0ff42590b25703c0acb3de3d3f4ede0410" + integrity sha512-t0hLfiEKfMUoqhG+U1oid7Pva4bbDPHYfJNiB7BiIjRkj1pyC++4N3huJfqY6aRH6VTB0rvtzQwjM4K6qpfOig== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"