diff --git a/front/API.ts b/front/API.ts index dc1f3fe..1b17bf4 100644 --- a/front/API.ts +++ b/front/API.ts @@ -17,13 +17,15 @@ export type AccessToken = string; type FetchParams = { route: string; body?: Object; - method?: 'GET' | 'POST' | 'DELETE' + method?: 'GET' | 'POST' | 'DELETE', + // If true, No JSON parsing is done, the raw response's content is returned + raw?: true; } const dummyIllustration = "https://i.discogs.com/syRCX8NaLwK2SMk8X6TVU_DWc8RRqE4b-tebAQ6kVH4/rs:fit/g:sm/q:90/h:600/w:600/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTgyNTQz/OC0xNjE3ODE0NDI2/LTU1MjUuanBlZw.jpeg"; // we will need the same thing for the scorometer API url -const baseAPIUrl = Platform.OS === 'web' ? '/api' : Constants.manifest?.extra?.apiUrl; +const baseAPIUrl = process.env.NODE_ENV != 'development' && Platform.OS === 'web' ? '/api' : Constants.manifest?.extra?.apiUrl; export default class API { @@ -37,6 +39,9 @@ export default class API { body: JSON.stringify(params.body), method: params.method ?? 'GET' }); + if (params.raw) { + return response.arrayBuffer(); + } const body = await response.text(); try { @@ -110,6 +115,16 @@ export default class API { route: `/song/${songId}` }); } + /** + * Retrive a song's midi partition + * @param songId the id to find the song + */ + public static async getSongMidi(songId: number): Promise { + return API.fetch({ + route: `/song/${songId}/midi`, + raw: true, + }); + } /** * Retrive an artist diff --git a/front/package.json b/front/package.json index 6a283fe..e25b459 100644 --- a/front/package.json +++ b/front/package.json @@ -39,6 +39,7 @@ "install": "^0.13.0", "jest": "^26.6.3", "jest-expo": "^45.0.1", + "midi-player-js": "^2.0.16", "moti": "^0.22.0", "native-base": "^3.4.17", "react": "18.1.0", @@ -56,6 +57,7 @@ "react-redux": "^8.0.2", "react-timer-hook": "^3.0.5", "redux-persist": "^6.0.0", + "soundfont-player": "^0.12.0", "yup": "^0.32.11" }, "devDependencies": { diff --git a/front/views/PlayView.tsx b/front/views/PlayView.tsx index 2d350fb..0fe0743 100644 --- a/front/views/PlayView.tsx +++ b/front/views/PlayView.tsx @@ -4,13 +4,14 @@ import * as ScreenOrientation from 'expo-screen-orientation'; import { Box, Center, Column, IconButton, Progress, Row, View, useToast } from 'native-base'; import { Ionicons } from "@expo/vector-icons"; import { useNavigation } from '@react-navigation/native'; -import { useQuery } from 'react-query'; +import { useQuery, useQueryClient } from 'react-query'; import API from '../API'; import LoadingComponent from '../components/Loading'; import Constants from 'expo-constants'; import { useStopwatch } from 'react-timer-hook'; -import PartitionVisualizer from '../components/PartitionVisualizer/PartitionVisualizer'; import SlideView from '../components/PartitionVisualizer/SlideView'; +import MidiPlayer from 'midi-player-js'; +import SoundFont from 'soundfont-player'; type PlayViewProps = { songId: number @@ -18,17 +19,21 @@ type PlayViewProps = { const PlayView = ({ songId }: PlayViewProps) => { const navigation = useNavigation(); - const song = useQuery(['song'], () => API.getSong(songId)); + const queryClient = useQueryClient(); + const song = useQuery(['song', songId], () => API.getSong(songId)); const toast = useToast(); const webSocket = useRef(); const timer = useStopwatch({ autoStart: false }); - const [paused, setPause] = useState(); - const partitionRessources = useQuery(["partition"], () => + const [paused, setPause] = useState(true); + const [midiPlayer, setMidiPlayer] = useState(); + + const partitionRessources = useQuery(["partition", songId], () => API.getPartitionRessources(songId) ); const onPause = () => { timer.pause(); + midiPlayer?.pause(); setPause(true); webSocket.current?.send(JSON.stringify({ type: "pause", @@ -38,6 +43,7 @@ const PlayView = ({ songId }: PlayViewProps) => { } const onResume = () => { setPause(false); + midiPlayer?.play(); timer.start(); webSocket.current?.send(JSON.stringify({ type: "pause", @@ -47,15 +53,11 @@ const PlayView = ({ songId }: PlayViewProps) => { } const onEnd = () => { webSocket.current?.close(); + midiPlayer?.destroy(); } const onMIDISuccess = (access) => { const inputs = access.inputs; - webSocket.current?.send(JSON.stringify({ - type: "start", - paused: false, - time: Date.now() - })); if (inputs.size < 2) { toast.show({ description: 'No MIDI Keyboard found' }); @@ -69,7 +71,6 @@ const PlayView = ({ songId }: PlayViewProps) => { type: "start", name: "clair-de-lune" /*song.data.id*/, })); - timer.start(); }; webSocket.current.onmessage = (message) => { try { @@ -83,7 +84,6 @@ const PlayView = ({ songId }: PlayViewProps) => { } } - setPause(false); inputs.forEach((input) => { if (inputIndex != 0) { return; @@ -100,6 +100,18 @@ const PlayView = ({ songId }: PlayViewProps) => { } inputIndex++; }); + Promise.all([ + queryClient.fetchQuery(['song', songId, 'midi'], () => API.getSongMidi(songId)), + SoundFont.instrument(new AudioContext(), 'electric_piano_1'), + ]).then(([midiFile, audioController]) => { + const player = new MidiPlayer.Player((event) => { + if (event['noteName']) { + audioController.play(event['noteName']); + } + }); + player.loadArrayBuffer(midiFile); + setMidiPlayer(player); + }) } const onMIDIFailure = () => { toast.show({ description: `Failed to get MIDI access` }); @@ -142,11 +154,11 @@ const PlayView = ({ songId }: PlayViewProps) => { }}/> { - if (paused == true) { + if (paused) { onResume(); - } else if (paused === false) { + } else { onPause(); } }}/> diff --git a/front/yarn.lock b/front/yarn.lock index 861c651..fe8fee6 100644 --- a/front/yarn.lock +++ b/front/yarn.lock @@ -5242,6 +5242,11 @@ address@^1.0.1: resolved "https://registry.yarnpkg.com/address/-/address-1.2.2.tgz#2b5248dac5485a6390532c6a517fda2e3faac89e" integrity sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA== +adsr@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/adsr/-/adsr-1.0.1.tgz#a7bc08e5ef8a71e6364abc96fce7df1c44881cc3" + integrity sha512-thr9LK4jxApOzBA33IWOA83bXJFbyfbeozpHXyrMQOIhUni198uRxXqDhobW0S/51iokqty2Yz2WbLZbE6tntQ== + agent-base@6: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" @@ -5659,6 +5664,11 @@ atob@^2.1.2: resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== +audio-loader@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/audio-loader/-/audio-loader-0.5.0.tgz#9c125d1b25c33cd9626084054d9f6b7f31ddc908" + integrity sha512-mEoYRjZhqkBSen/X9i2PNosqvafEsur8bI5MNoPr0wsJu9Nzlul3Yv1elYeMPsXxTxYhXLY8AZlScBvaK4mydg== + autoprefixer@^9.8.6: version "9.8.8" resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.8.tgz#fd4bd4595385fa6f06599de749a4d5f7a474957a" @@ -12556,6 +12566,16 @@ microseconds@0.2.0: resolved "https://registry.yarnpkg.com/microseconds/-/microseconds-0.2.0.tgz#233b25f50c62a65d861f978a4a4f8ec18797dc39" integrity sha512-n7DHHMjR1avBbSpsTBj6fmMGh2AGrifVV4e+WYc3Q9lO+xnSZ3NyhcBND3vzzatt05LFhoKFRxrIyklmLlUtyA== +midi-player-js@^2.0.16: + version "2.0.16" + resolved "https://registry.yarnpkg.com/midi-player-js/-/midi-player-js-2.0.16.tgz#41167859e3f430e55eeb962887cb498726d6c570" + integrity sha512-Y1yCRvvSjJjT5J4U8T4XTCDF1FLXtw8Otvq5BAmIob/2cj10aQUDrPDFByTWeuMRPu6/nLhusROc1DuTLCzRnw== + +midimessage@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/midimessage/-/midimessage-1.0.5.tgz#ad99f04d863a053a2563d553c5bf35070b48802c" + integrity sha512-MPJ2tDupFOfZB5/PLp8fri1IS4fd9hPj0Bio//FBhWRQ+TsJA7/49CF1aJyraDxa0Jq8zMHAwrwXl2GINvLvgw== + miller-rabin@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/miller-rabin/-/miller-rabin-4.0.1.tgz#f080351c865b0dc562a8462966daa53543c78a4d" @@ -13088,6 +13108,16 @@ normalize-url@^6.0.1: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== +note-parser@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/note-parser/-/note-parser-1.1.0.tgz#12e9f17e51450ec994f1364a01982c22667b8e6b" + integrity sha512-YTqWQBsRp40EFrEznnkGtmx68gcgOQ8CdoBspqGBA3G1/4mJwIYbDe/vuNpX3oGX2DhP7b1dBgTmj7p3Zr0P1Q== + +note-parser@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/note-parser/-/note-parser-2.0.1.tgz#2438fd57a46894b402b3a2071798660129c8fbc1" + integrity sha512-w9o6Fv46y3NsFxeezTZSmftBtUM/ypme6iZWVrTJvvsD5RN+w0XNDePWtfreNrZFL3jSjBFhadPoXb+pJO4UdA== + npm-package-arg@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/npm-package-arg/-/npm-package-arg-7.0.0.tgz#52cdf08b491c0c59df687c4c925a89102ef794a5" @@ -15481,6 +15511,15 @@ safe-regex@^1.1.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sample-player@^0.5.5: + version "0.5.5" + resolved "https://registry.yarnpkg.com/sample-player/-/sample-player-0.5.5.tgz#bc35bea3449c6fa972528f022a9bbc2872195637" + integrity sha512-VQ9pXPJ1m/eTH8QK6OQ8Dn/HSVToNyY9w9vnv+y/yjkJeRm87tJ/gBEm66jItfSLhKe6VG1DfX8+oT+Mg7QUpg== + dependencies: + adsr "^1.0.0" + midimessage "^1.0.5" + note-parser "^1.1.0" + sane@^4.0.3: version "4.1.0" resolved "https://registry.yarnpkg.com/sane/-/sane-4.1.0.tgz#ed881fd922733a6c461bc189dc2b6c006f3ffded" @@ -15903,6 +15942,15 @@ sockjs@0.3.20: uuid "^3.4.0" websocket-driver "0.6.5" +soundfont-player@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/soundfont-player/-/soundfont-player-0.12.0.tgz#2b26149f28aba471d2285d3df9a2e1e5793ceaf1" + integrity sha512-8BJIsAt7h1PK3thSZDgF6zecgGhYkK74JnZO8WRZi3h34qG6H/DYlnv7cpRvL7Q9C8N6qld4Qwj7nJsX1gYjEA== + dependencies: + audio-loader "^0.5.0" + note-parser "^2.0.0" + sample-player "^0.5.5" + source-list-map@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34"