feat(packages): add mux media with src parsing, structured source, and storyboards (#1850)

Co-authored-by: Claude <noreply@anthropic.com>

packages/react/src/media/mux-audio/index.tsx

@@ -2,9 +2,10 @@
 
 import { GoogleCast } from '@videojs/core/dom/media/google-cast';
 import type { HlsMediaProps } from '@videojs/core/dom/media/hls-js';
-import { HlsJsMedia, hlsMediaDefaultProps } from '@videojs/core/dom/media/hls-js';
+import { hlsMediaDefaultProps } from '@videojs/core/dom/media/hls-js';
 import { addComponent } from '@videojs/core/dom/media/media-host';
-import { MuxData } from '@videojs/core/dom/media/mux';
+import type { MuxMediaProps } from '@videojs/core/dom/media/mux';
+import { MuxData, MuxMedia, muxMediaDefaultProps } from '@videojs/core/dom/media/mux';
 import type { AudioHTMLAttributes, ReactNode } from 'react';
 import { forwardRef } from 'react';
 import { useAttachMedia } from '../../utils/use-attach-media';
@@ -13,19 +14,22 @@ import { useMediaInstance } from '../../utils/use-media-instance';
 import { useSyncProps } from '../../utils/use-sync-props';
 
 export interface MuxAudioProps
-  extends Omit<AudioHTMLAttributes<HTMLAudioElement>, keyof HlsMediaProps>,
-    Partial<HlsMediaProps> {
+  extends Omit<AudioHTMLAttributes<HTMLAudioElement>, keyof HlsMediaProps | keyof MuxMediaProps>,
+    Partial<HlsMediaProps>,
+    Partial<MuxMediaProps> {
   children?: ReactNode;
 }
 
+const muxAudioDefaultProps: HlsMediaProps & MuxMediaProps = { ...hlsMediaDefaultProps, ...muxMediaDefaultProps };
+
 export const MuxAudio = forwardRef<HTMLAudioElement, MuxAudioProps>(function MuxAudio({ children, ...props }, ref) {
-  const media = useMediaInstance(HlsJsMedia, (media) => {
+  const media = useMediaInstance(MuxMedia, (media) => {
     addComponent(media, new MuxData({ playerSoftwareName: 'mux-audio' }));
     addComponent(media, new GoogleCast());
   });
   const attachRef = useAttachMedia(media);
   const composedRef = useComposedRefs(attachRef, ref);
-  const htmlProps = useSyncProps(media, props, hlsMediaDefaultProps);
+  const htmlProps = useSyncProps(media, props, muxAudioDefaultProps);
 
   return (
     <audio ref={composedRef} {...htmlProps}>

packages/react/src/media/mux-video/index.tsx

@@ -2,33 +2,38 @@
 
 import { GoogleCast } from '@videojs/core/dom/media/google-cast';
 import type { HlsMediaProps } from '@videojs/core/dom/media/hls-js';
-import { HlsJsMedia, hlsMediaDefaultProps } from '@videojs/core/dom/media/hls-js';
+import { hlsMediaDefaultProps, StreamTypes } from '@videojs/core/dom/media/hls-js';
 import { addComponent } from '@videojs/core/dom/media/media-host';
-import { MuxData } from '@videojs/core/dom/media/mux';
+import type { MuxMediaProps } from '@videojs/core/dom/media/mux';
+import { MuxData, MuxMedia, muxMediaDefaultProps } from '@videojs/core/dom/media/mux';
 import type { ReactNode, VideoHTMLAttributes } from 'react';
-import { forwardRef } from 'react';
+import { forwardRef, useCallback, useSyncExternalStore } from 'react';
 import { useAttachMedia } from '../../utils/use-attach-media';
 import { useComposedRefs } from '../../utils/use-composed-refs';
 import { useMediaInstance } from '../../utils/use-media-instance';
 import { useSyncProps } from '../../utils/use-sync-props';
 
 export interface MuxVideoProps
-  extends Omit<VideoHTMLAttributes<HTMLVideoElement>, keyof HlsMediaProps>,
-    Partial<HlsMediaProps> {
+  extends Omit<VideoHTMLAttributes<HTMLVideoElement>, keyof HlsMediaProps | keyof MuxMediaProps>,
+    Partial<HlsMediaProps>,
+    Partial<MuxMediaProps> {
   children?: ReactNode;
 }
 
+const muxVideoDefaultProps: HlsMediaProps & MuxMediaProps = { ...hlsMediaDefaultProps, ...muxMediaDefaultProps };
+
 export const MuxVideo = forwardRef<HTMLVideoElement, MuxVideoProps>(function MuxVideo({ children, ...props }, ref) {
-  const media = useMediaInstance(HlsJsMedia, (media) => {
+  const media = useMediaInstance(MuxMedia, (media) => {
     addComponent(media, new MuxData({ playerSoftwareName: 'mux-video' }));
     addComponent(media, new GoogleCast());
   });
   const attachRef = useAttachMedia(media);
   const composedRef = useComposedRefs(attachRef, ref);
-  const htmlProps = useSyncProps(media, props, hlsMediaDefaultProps);
+  const htmlProps = useSyncProps(media, props, muxVideoDefaultProps);
 
   return (
     <video ref={composedRef} {...htmlProps}>
+      <MuxStoryboard media={media} />
       {children}
     </video>
   );
@@ -37,3 +42,39 @@ export const MuxVideo = forwardRef<HTMLVideoElement, MuxVideoProps>(function Mux
 export namespace MuxVideo {
   export type Props = MuxVideoProps;
 }
+
+// Renders the storyboard track in its own component so media changes don't re-render the whole media component.
+function MuxStoryboard({ media }: { media: MuxMedia }) {
+  const subscribe = useCallback(
+    (onChange: () => void) => {
+      // `useSyncProps` writes `src` / `source` during render and those setters
+      // dispatch `sourcechange` synchronously. Defer (and coalesce) notifications
+      // so we never schedule an update while another component is rendering.
+      let cancelled = false;
+      let scheduled = false;
+      const notify = () => {
+        if (scheduled) return;
+        scheduled = true;
+        queueMicrotask(() => {
+          scheduled = false;
+          if (!cancelled) onChange();
+        });
+      };
+      media.addEventListener('streamtypechange', notify);
+      media.addEventListener('sourcechange', notify);
+      return () => {
+        cancelled = true;
+        media.removeEventListener('streamtypechange', notify);
+        media.removeEventListener('sourcechange', notify);
+      };
+    },
+    [media]
+  );
+
+  // The stream type is detected at runtime and live streams have no storyboard.
+  const getSnapshot = () => (media.streamType === StreamTypes.LIVE ? '' : media.storyboard);
+  const src = useSyncExternalStore(subscribe, getSnapshot, getSnapshot);
+
+  if (!src) return null;
+  return <track kind="metadata" label="thumbnails" src={src} default />;
+}

packages/react/src/media/tests/mux-video.test.tsx

@@ -1,5 +1,6 @@
 import { render } from '@testing-library/react';
-import { MuxData } from '@videojs/core/dom/media/mux';
+import { HlsJsMedia } from '@videojs/core/dom/media/hls-js';
+import { MuxData, MuxMedia } from '@videojs/core/dom/media/mux';
 import { describe, expect, it, vi } from 'vitest';
 import { MuxVideo } from '../mux-video';
 
@@ -31,4 +32,107 @@ describe('MuxVideo', () => {
 
     reinit.mockRestore();
   });
+
+  it('derives the media src from the source prop', () => {
+    const src = vi.spyOn(HlsJsMedia.prototype, 'src', 'set');
+
+    render(<MuxVideo source={{ playbackId: 'abc123' }} />);
+
+    expect(src).toHaveBeenCalledWith('https://stream.mux.com/abc123.m3u8');
+
+    src.mockRestore();
+  });
+
+  it('applies the customDomain and playback params to src', () => {
+    const src = vi.spyOn(HlsJsMedia.prototype, 'src', 'set');
+
+    render(
+      <MuxVideo source={{ playbackId: 'abc123', customDomain: 'example.com', playback: { maxResolution: '1080p' } }} />
+    );
+
+    const url = new URL(src.mock.calls[src.mock.calls.length - 1]![0]);
+    expect(url.host).toBe('stream.example.com');
+    expect(url.searchParams.get('max_resolution')).toBe('1080p');
+
+    src.mockRestore();
+  });
+
+  it('omits playback params from src when a playback token is set', () => {
+    const src = vi.spyOn(HlsJsMedia.prototype, 'src', 'set');
+
+    render(<MuxVideo source={{ playbackId: 'abc123', playback: { token: 'jwt', assetStartTime: 3 } }} />);
+
+    const url = new URL(src.mock.calls[src.mock.calls.length - 1]![0]);
+    expect(url.searchParams.get('token')).toBe('jwt');
+    expect(url.searchParams.has('asset_start_time')).toBe(false);
+
+    src.mockRestore();
+  });
+
+  it('adds a storyboard track inferred from the source prop', () => {
+    const { container } = render(<MuxVideo source={{ playbackId: 'abc123' }} />);
+
+    const track = container.querySelector('track');
+    expect(track?.kind).toBe('metadata');
+    expect(track?.getAttribute('src')).toBe('https://image.mux.com/abc123/storyboard.vtt?format=webp');
+  });
+
+  it('adds a storyboard track inferred from a Mux stream src', () => {
+    const { container } = render(<MuxVideo src="https://stream.mux.com/abc123.m3u8" />);
+
+    expect(container.querySelector('track')?.getAttribute('src')).toBe(
+      'https://image.mux.com/abc123/storyboard.vtt?format=webp'
+    );
+  });
+
+  it('does not add a storyboard track for a non-Mux src', () => {
+    const { container } = render(<MuxVideo src="https://example.com/video.m3u8" />);
+
+    expect(container.querySelector('track')).toBeNull();
+  });
+
+  it('updates the storyboard track when the source changes without render-phase warnings', () => {
+    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    const { container, rerender } = render(<MuxVideo source={{ playbackId: 'abc123' }} />);
+    rerender(<MuxVideo source={{ playbackId: 'xyz789' }} />);
+
+    expect(container.querySelector('track')?.getAttribute('src')).toBe(
+      'https://image.mux.com/xyz789/storyboard.vtt?format=webp'
+    );
+    // Guards against "Cannot update a component while rendering a different component".
+    expect(consoleError).not.toHaveBeenCalled();
+
+    consoleError.mockRestore();
+  });
+
+  it('clears a storyboard override when the prop is removed', () => {
+    const { container, rerender } = render(
+      <MuxVideo source={{ playbackId: 'abc123' }} storyboard="https://image.mux.com/other/storyboard.vtt" />
+    );
+
+    expect(container.querySelector('track')?.getAttribute('src')).toBe('https://image.mux.com/other/storyboard.vtt');
+
+    rerender(<MuxVideo source={{ playbackId: 'abc123' }} />);
+
+    expect(container.querySelector('track')?.getAttribute('src')).toBe(
+      'https://image.mux.com/abc123/storyboard.vtt?format=webp'
+    );
+  });
+
+  it('does not add a storyboard track for live streams', () => {
+    const streamType = vi.spyOn(MuxMedia.prototype, 'streamType', 'get').mockReturnValue('live');
+
+    const { container } = render(<MuxVideo source={{ playbackId: 'abc123' }} />);
+
+    expect(container.querySelector('track')).toBeNull();
+
+    streamType.mockRestore();
+  });
+
+  it('does not sync the source thumbnail to the media poster', () => {
+    const { container } = render(<MuxVideo source={{ playbackId: 'abc123', thumbnail: { time: 5 } }} />);
+
+    expect(container.querySelector('video')?.getAttribute('poster')).toBeNull();
+  });
 });

packages/react/src/utils/tests/use-sync-props.test.tsx

@@ -0,0 +1,104 @@
+import { cleanup, renderHook } from '@testing-library/react';
+import { afterEach, describe, expect, it } from 'vitest';
+import { useSyncProps } from '../use-sync-props';
+
+afterEach(cleanup);
+
+interface TargetProps {
+  src: string;
+  volume: number | undefined;
+}
+
+const defaults: TargetProps = { src: '', volume: 1 };
+
+describe('useSyncProps', () => {
+  it('writes props onto the target and returns the rest', () => {
+    const target: TargetProps = { ...defaults };
+
+    const { result } = renderHook(({ props }) => useSyncProps(target, props, defaults), {
+      initialProps: { props: { src: 'video.mp4', id: 'player' } },
+    });
+
+    expect(target.src).toBe('video.mp4');
+    expect(result.current).toEqual({ id: 'player' });
+  });
+
+  it('resets props back to defaults when they change to undefined on a re-render', () => {
+    const target: TargetProps = { ...defaults };
+
+    const { rerender } = renderHook(({ props }) => useSyncProps(target, props, defaults), {
+      initialProps: { props: { volume: 0.5 } as Partial<TargetProps> },
+    });
+
+    expect(target.volume).toBe(0.5);
+
+    rerender({ props: { volume: undefined } });
+
+    expect(target.volume).toBe(1);
+  });
+
+  it('treats undefined like an absent prop and never touches unsynced target values', () => {
+    const target: TargetProps = { ...defaults, volume: 0.5 };
+
+    renderHook(() => useSyncProps(target, { volume: undefined }, defaults));
+
+    expect(target.volume).toBe(0.5);
+  });
+
+  it('does not let an undefined prop wipe a value derived from another prop', () => {
+    // Mirrors MuxMedia: setting `source` derives `src`, resetting `src` clears `source`.
+    const derivedDefaults: { src: string | undefined; source: { id: string } | null } = { src: '', source: null };
+    const target = {
+      _src: '' as string | undefined,
+      _source: null as { id: string } | null,
+      get src() {
+        return this._src;
+      },
+      set src(value: string | undefined) {
+        this._src = value;
+        this._source = value ? { id: value } : null;
+      },
+      get source() {
+        return this._source;
+      },
+      set source(value: { id: string } | null) {
+        this._source = value;
+        this._src = value ? value.id : '';
+      },
+    };
+
+    // `source` before `src` in key order — the reset must not run after it applies.
+    renderHook(() => useSyncProps(target, { source: { id: 'abc' }, src: undefined }, derivedDefaults));
+
+    expect(target.source).toEqual({ id: 'abc' });
+    expect(target.src).toBe('abc');
+  });
+
+  it('resets props back to defaults when they are omitted on a re-render', () => {
+    const target: TargetProps = { ...defaults };
+
+    const { rerender } = renderHook(({ props }) => useSyncProps(target, props, defaults), {
+      initialProps: { props: { src: 'video.mp4', volume: 0.5 } as Partial<TargetProps> },
+    });
+
+    expect(target.volume).toBe(0.5);
+
+    rerender({ props: { src: 'video.mp4' } });
+
+    expect(target.volume).toBe(1);
+    expect(target.src).toBe('video.mp4');
+  });
+
+  it('does not touch target values that were never passed as props', () => {
+    const target: TargetProps = { ...defaults, volume: 0.5 };
+
+    const { rerender } = renderHook(({ props }) => useSyncProps(target, props, defaults), {
+      initialProps: { props: { src: 'video.mp4' } as Partial<TargetProps> },
+    });
+
+    rerender({ props: { src: 'video.mp4' } });
+
+    // `volume` was set outside of props; omitting it from props never resets it.
+    expect(target.volume).toBe(0.5);
+  });
+});

packages/react/src/utils/use-sync-props.ts

@@ -1,4 +1,5 @@
 import { isUndefined } from '@videojs/utils/predicate';
+import { useRef } from 'react';
 
 export function useSyncProps<Props extends object, Rest extends Record<string, unknown>>(
   target: Props,
@@ -6,15 +7,32 @@ export function useSyncProps<Props extends object, Rest extends Record<string, u
   defaults: Props
 ): Omit<Rest, keyof Props> {
   const rest: Record<string, unknown> = {};
+  const synced = new Set<string>();
+  const prevSyncedRef = useRef<Set<string> | null>(null);
+
+  const sync = (key: string, value: unknown) => {
+    if (target[key as keyof Props] !== value) target[key as keyof Props] = value as Props[keyof Props];
+  };
+
+  // Reset props the consumer stopped passing (or passed as `undefined`) back to
+  // their defaults before applying the current ones, so a reset can never wipe a
+  // value another prop derives in the same render (e.g. `source` deriving `src`).
+  // Mirrors react-dom removing absent attributes.
+  for (const key of prevSyncedRef.current ?? []) {
+    if (isUndefined((props as Record<string, unknown>)[key])) sync(key, (defaults as Record<string, unknown>)[key]);
+  }
 
   for (const key in props) {
     if (key in defaults) {
-      const value = isUndefined(props[key]) ? (defaults as Record<string, unknown>)[key] : props[key];
-      if (target[key as keyof typeof target] !== value) target[key as keyof typeof target] = value as any;
+      if (isUndefined(props[key])) continue;
+      synced.add(key);
+      sync(key, props[key]);
     } else {
       rest[key] = props[key];
     }
   }
 
+  prevSyncedRef.current = synced;
+
   return rest as Omit<Rest, keyof Props>;
 }