feat: add subtitles handling + captions core (#692)

packages/core/src/dom/media/custom-media-element/index.ts

@@ -129,7 +129,7 @@ declare class CustomAudioElementClass extends HTMLAudioElement implements HTMLAu
   static getTemplateHTML: typeof getAudioTemplateHTML;
   static shadowRootOptions: ShadowRootInit;
   static Events: string[];
-  readonly nativeEl: HTMLAudioElement;
+  readonly target: HTMLAudioElement;
   attributeChangedCallback(attrName: string, oldValue?: string | null, newValue?: string | null): void;
   connectedCallback(): void;
   disconnectedCallback(): void;
@@ -142,7 +142,7 @@ declare class CustomVideoElementClass extends HTMLVideoElement implements HTMLVi
   static getTemplateHTML: typeof getVideoTemplateHTML;
   static shadowRootOptions: ShadowRootInit;
   static Events: string[];
-  readonly nativeEl: HTMLVideoElement;
+  readonly target: HTMLVideoElement;
   attributeChangedCallback(attrName: string, oldValue?: string | null, newValue?: string | null): void;
   connectedCallback(): void;
   disconnectedCallback(): void;
@@ -239,7 +239,7 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
 
     // Private fields
     #isInit = false;
-    #nativeEl: HTMLVideoElement | HTMLAudioElement | null = null;
+    #target: HTMLVideoElement | HTMLAudioElement | null = null;
     #childMap = new Map<MediaChild, MediaChild>();
     #childObserver?: MutationObserver;
 
@@ -249,7 +249,7 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
         const val = this.getAttribute(attr);
         return val === null ? false : val === '' ? true : val;
       }
-      return this.nativeEl?.[prop as keyof typeof this.nativeEl];
+      return this.target?.[prop as keyof typeof this.target];
     }
 
     set(prop: string, val: any): void {
@@ -263,15 +263,15 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
         return;
       }
 
-      if (this.nativeEl) {
+      if (this.target) {
         // @ts-expect-error
-        this.nativeEl[prop as keyof typeof this.nativeEl] = val;
+        this.target[prop as keyof typeof this.target] = val;
       }
     }
 
     call(prop: string, ...args: any[]): any {
-      const nativeFn = this.nativeEl?.[prop as keyof typeof this.nativeEl] as ((...args: any[]) => any) | undefined;
-      return nativeFn?.apply(this.nativeEl, args);
+      const nativeFn = this.target?.[prop as keyof typeof this.target] as ((...args: any[]) => any) | undefined;
+      return nativeFn?.apply(this.target, args);
     }
 
     // If the custom element is defined before the custom element's HTML is parsed
@@ -279,10 +279,10 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
     // Wait until initializing in the attributeChangedCallback or
     // connectedCallback or accessing any properties.
 
-    get nativeEl() {
+    get target() {
       this.#init();
       return (
-        this.#nativeEl ??
+        this.#target ??
         this.querySelector(':scope > [slot=media]') ??
         this.querySelector(tag) ??
         this.shadowRoot?.querySelector(tag) ??
@@ -290,8 +290,8 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
       );
     }
 
-    set nativeEl(val: HTMLVideoElement | HTMLAudioElement | null) {
-      this.#nativeEl = val;
+    set target(val: HTMLVideoElement | HTMLAudioElement | null) {
+      this.#target = val;
     }
 
     get defaultMuted() {
@@ -323,7 +323,7 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
       // Neither Chrome or Firefox support setting the muted attribute
       // after using document.createElement.
       // Get around this by setting the muted property manually.
-      this.nativeEl!.muted = this.hasAttribute('muted');
+      this.target!.muted = this.hasAttribute('muted');
 
       for (const prop of nativeElProps) {
         // @ts-expect-error
@@ -340,7 +340,7 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
     }
 
     handleEvent(event: Event): void {
-      if (event.target === this.nativeEl) {
+      if (event.target === this.target) {
         this.dispatchEvent(new CustomEvent(event.type, { detail: (event as CustomEvent).detail }));
       }
     }
@@ -361,7 +361,7 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
           this.#childMap.set(el, clone);
           this.#childObserver?.observe(el, { attributes: true });
         }
-        this.nativeEl?.append(clone);
+        this.target?.append(clone);
         this.#enableDefaultTrack(clone as HTMLTrackElement);
       });
 
@@ -426,9 +426,9 @@ export function CustomMediaMixin<T extends Constructor<HTMLElement>>(
       }
 
       if (newValue === null) {
-        this.nativeEl?.removeAttribute(attrName);
-      } else if (this.nativeEl?.getAttribute(attrName) !== newValue) {
-        this.nativeEl?.setAttribute(attrName, newValue);
+        this.target?.removeAttribute(attrName);
+      } else if (this.target?.getAttribute(attrName) !== newValue) {
+        this.target?.setAttribute(attrName, newValue);
       }
     }
 

packages/core/src/dom/media/hls/index.ts

@@ -1,11 +1,24 @@
 import Hls from 'hls.js';
 
-import { type MediaDelegate, MediaDelegateMixin } from '../../core/media/delegate';
-import { MediaProxyMixin } from '../../core/media/proxy';
-import { CustomMediaMixin } from './custom-media-element';
+import { type MediaDelegate, MediaDelegateMixin } from '../../../core/media/delegate';
+import { MediaProxyMixin } from '../../../core/media/proxy';
+import { CustomMediaMixin } from '../custom-media-element';
+import { HlsMediaTextTracksMixin } from './text-tracks';
 
-export class HlsMediaDelegate implements MediaDelegate {
-  #engine = new Hls();
+const defaultConfig = {
+  backBufferLength: 30,
+  renderTextTracksNatively: false,
+  liveDurationInfinity: true,
+  capLevelToPlayerSize: true,
+  capLevelOnFPSDrop: true,
+};
+
+export class HlsMediaDelegateBase implements MediaDelegate {
+  #engine = new Hls(defaultConfig);
+
+  get engine(): Hls {
+    return this.#engine;
+  }
 
   attach(target: EventTarget): void {
     this.#engine.attachMedia(target as HTMLMediaElement);
@@ -24,6 +37,8 @@ export class HlsMediaDelegate implements MediaDelegate {
   }
 }
 
+const HlsMediaDelegate = HlsMediaTextTracksMixin(HlsMediaDelegateBase);
+
 // This is used by the web component because it needs to extend HTMLElement!
 export class HlsCustomMedia extends MediaDelegateMixin(
   CustomMediaMixin(globalThis.HTMLElement ?? class {}, { tag: 'video' }),

packages/core/src/dom/media/hls/text-tracks.ts

@@ -0,0 +1,178 @@
+import { listen } from '@videojs/utils/dom';
+import type { Constructor } from '@videojs/utils/types';
+import type { CuesParsedData, NonNativeTextTracksData } from 'hls.js';
+import Hls from 'hls.js';
+
+interface HlsEngineHost {
+  readonly engine: Hls;
+  attach?(target: EventTarget): void;
+  detach?(): void;
+}
+
+/**
+ * Bridges hls.js non-native text tracks to native `<track>` elements so the
+ * rest of the player can treat them like any other text track.
+ *
+ * When `renderTextTracksNatively: false`, hls.js fires
+ * `NON_NATIVE_TEXT_TRACKS_FOUND` with track metadata and `CUES_PARSED` with
+ * VTTCues. This mixin creates `<track>` elements on the media target and
+ * forwards cues into them. It also syncs user track-mode changes back to
+ * hls.js via `engine.subtitleTrack`.
+ */
+export function HlsMediaTextTracksMixin<Base extends Constructor<HlsEngineHost>>(BaseClass: Base) {
+  class HlsMediaTextTracks extends (BaseClass as Constructor<HlsEngineHost>) {
+    #disconnect: AbortController | null = null;
+    #target: HTMLMediaElement | null = null;
+
+    attach(target: EventTarget): void {
+      super.attach?.(target);
+      this.#target = target as HTMLMediaElement;
+      this.#connect();
+    }
+
+    detach(): void {
+      this.#disconnect?.abort();
+      this.#disconnect = null;
+      this.#target = null;
+      super.detach?.();
+    }
+
+    #connect(): void {
+      this.#disconnect?.abort();
+      this.#disconnect = new AbortController();
+
+      const { signal } = this.#disconnect;
+      const { engine } = this;
+      const media = this.#target!;
+
+      const onTracksFound = (_event: string, data: NonNativeTextTracksData) => {
+        this.#clearTracks();
+
+        for (const trackObj of data.tracks) {
+          const baseTrackObj = trackObj.subtitleTrack ?? trackObj.closedCaptions;
+          const idx = engine.subtitleTracks.findIndex(({ lang, name, type }) => {
+            return lang === baseTrackObj?.lang && name === trackObj.label && type.toLowerCase() === trackObj.kind;
+          });
+
+          // NOTE: Undocumented method for determining identifier by hls.js. Relied on for
+          // ensuring CUES_PARSED events can identify and apply cues to the appropriate track (CJP).
+          // See: https://github.com/video-dev/hls.js/blob/master/src/controller/timeline-controller.ts#L640
+          const id = (trackObj._id ?? trackObj.default) ? 'default' : `${trackObj.kind}${idx}`;
+
+          addTextTrack(media, trackObj.kind as TextTrackKind, trackObj.label, baseTrackObj?.lang, id, trackObj.default);
+        }
+      };
+
+      const onCuesParsed = (_event: string, { track, cues }: CuesParsedData) => {
+        const textTrack = media.textTracks.getTrackById(track);
+        if (!textTrack) return;
+
+        const disabled = textTrack.mode === 'disabled';
+        if (disabled) {
+          textTrack.mode = 'hidden';
+        }
+
+        cues.forEach((cue: VTTCue) => {
+          if (textTrack.cues?.getCueById(cue.id)) return;
+          textTrack.addCue(cue);
+        });
+
+        if (disabled) {
+          textTrack.mode = 'disabled';
+        }
+      };
+
+      const onTextTrackChange = () => {
+        if (!engine.subtitleTracks.length) return;
+
+        const showingTrack = Array.from(media.textTracks).find((textTrack) => {
+          return textTrack.id && textTrack.mode === 'showing' && ['subtitles', 'captions'].includes(textTrack.kind);
+        });
+
+        if (!showingTrack) return;
+
+        const currentHlsTrack = engine.subtitleTracks[engine.subtitleTrack];
+
+        // If hls.subtitleTrack is -1 or its id changed compared to the one that is showing load the new subtitle track.
+        const hlsTrackId = !currentHlsTrack
+          ? undefined
+          : currentHlsTrack.default
+            ? 'default'
+            : `${engine.subtitleTracks[engine.subtitleTrack]?.type.toLowerCase()}${engine.subtitleTrack}`;
+
+        if (engine.subtitleTrack < 0 || showingTrack?.id !== hlsTrackId) {
+          const idx = engine.subtitleTracks.findIndex(({ lang, name, type, default: defaultTrack }) => {
+            return (
+              (showingTrack.id === 'default' && defaultTrack) ||
+              (lang === showingTrack.language &&
+                name === showingTrack.label &&
+                type.toLowerCase() === showingTrack.kind)
+            );
+          });
+          // After the subtitleTrack is set here, hls.js will load the playlist and CUES_PARSED events will be fired below.
+          engine.subtitleTrack = idx;
+        }
+
+        if (showingTrack?.id === hlsTrackId) {
+          // Refresh the cues after a texttrack mode change to fix a Chrome bug causing the captions not to render.
+          if (showingTrack.cues) {
+            Array.from(showingTrack.cues).forEach((cue) => {
+              showingTrack.addCue(cue);
+            });
+          }
+        }
+      };
+
+      engine.on(Hls.Events.NON_NATIVE_TEXT_TRACKS_FOUND, onTracksFound);
+      engine.on(Hls.Events.CUES_PARSED, onCuesParsed);
+      listen(media.textTracks, 'change', onTextTrackChange, { signal });
+
+      signal.addEventListener(
+        'abort',
+        () => {
+          engine.off(Hls.Events.NON_NATIVE_TEXT_TRACKS_FOUND, onTracksFound);
+          engine.off(Hls.Events.CUES_PARSED, onCuesParsed);
+          this.#clearTracks();
+        },
+        { once: true }
+      );
+    }
+
+    #clearTracks(): void {
+      const trackEls = this.#target!.querySelectorAll('track[data-removeondestroy]');
+      trackEls.forEach((trackEl) => trackEl.remove());
+    }
+  }
+
+  return HlsMediaTextTracks as unknown as Base;
+}
+
+function addTextTrack(
+  mediaEl: HTMLMediaElement,
+  kind: TextTrackKind,
+  label: string,
+  lang?: string,
+  id?: string,
+  defaultTrack?: boolean
+): TextTrack {
+  const trackEl = document.createElement('track');
+  trackEl.kind = kind;
+  trackEl.label = label;
+  if (lang) {
+    // This attribute must be present if the element's kind attribute is in the subtitles state.
+    trackEl.srclang = lang;
+  }
+  if (id) {
+    trackEl.id = id;
+  }
+  if (defaultTrack) {
+    trackEl.default = true;
+  }
+  trackEl.track.mode = ['subtitles', 'captions'].includes(kind) ? 'disabled' : 'hidden';
+
+  // Add data attribute to identify tracks that should be removed when switching sources/destroying hls.js instance.
+  trackEl.setAttribute('data-removeondestroy', '');
+  mediaEl.append(trackEl);
+
+  return trackEl.track as TextTrack;
+}

packages/core/src/dom/store/features/tests/text-track.test.ts

@@ -16,6 +16,23 @@ function createVideo(): HTMLVideoElement {
   return document.createElement('video');
 }
 
+function mockTextTracks(video: HTMLVideoElement, tracks: TextTrack[]): void {
+  const list: Partial<TextTrackList> & Record<number, TextTrack> = { length: tracks.length };
+
+  for (const [index, track] of tracks.entries()) {
+    list[index] = track;
+  }
+
+  Object.defineProperty(video, 'textTracks', {
+    configurable: true,
+    value: list as TextTrackList,
+  });
+}
+
+function createMockTrack(kind: TextTrackKind, mode: TextTrackMode = 'disabled'): TextTrack {
+  return { kind, mode, label: '', language: '' } as TextTrack;
+}
+
 describe('textTrackFeature', () => {
   describe('initial state', () => {
     it('has empty initial state', () => {
@@ -26,6 +43,8 @@ describe('textTrackFeature', () => {
       expect(store.state.chaptersCues).toEqual([]);
       expect(store.state.thumbnailCues).toEqual([]);
       expect(store.state.thumbnailTrackSrc).toBeNull();
+      expect(store.state.subtitlesList).toEqual([]);
+      expect(store.state.subtitlesShowing).toBe(false);
     });
   });
 
@@ -120,6 +139,62 @@ describe('textTrackFeature', () => {
       }
     });
 
+    it('sets subtitlesShowing when a subtitles track is showing', () => {
+      const video = createVideo();
+      mockTextTracks(video, [createMockTrack('subtitles', 'showing')]);
+
+      const store = createStore<PlayerTarget>()(textTrackFeature);
+      store.attach({ media: video, container: null });
+
+      expect(store.state.subtitlesShowing).toBe(true);
+    });
+
+    it('exposes subtitlesList from captions/subtitles tracks', () => {
+      const video = createVideo();
+      const subtitlesTrack = { kind: 'subtitles', mode: 'showing', label: 'English', language: 'en' } as TextTrack;
+      const captionsTrack = { kind: 'captions', mode: 'disabled', label: 'CC', language: 'en' } as TextTrack;
+      mockTextTracks(video, [subtitlesTrack, captionsTrack, createMockTrack('metadata', 'showing')]);
+
+      const store = createStore<PlayerTarget>()(textTrackFeature);
+      store.attach({ media: video, container: null });
+
+      expect(store.state.subtitlesList).toEqual([
+        { kind: 'subtitles', label: 'English', language: 'en', mode: 'showing' },
+        { kind: 'captions', label: 'CC', language: 'en', mode: 'disabled' },
+      ]);
+    });
+
+    it('toggleSubtitles() enables and disables caption/subtitle tracks', () => {
+      const video = createVideo();
+      const subtitlesTrack = createMockTrack('subtitles');
+      const captionsTrack = createMockTrack('captions');
+      mockTextTracks(video, [subtitlesTrack, captionsTrack]);
+
+      const store = createStore<PlayerTarget>()(textTrackFeature);
+      store.attach({ media: video, container: null });
+
+      const enabled = store.state.toggleSubtitles();
+      expect(enabled).toBe(true);
+      expect(subtitlesTrack.mode).toBe('showing');
+      expect(captionsTrack.mode).toBe('showing');
+
+      const disabled = store.state.toggleSubtitles(false);
+      expect(disabled).toBe(false);
+      expect(subtitlesTrack.mode).toBe('disabled');
+      expect(captionsTrack.mode).toBe('disabled');
+    });
+
+    it('toggleSubtitles() returns false when no subtitle tracks exist', () => {
+      const video = createVideo();
+      const metadataTrack = createMockTrack('metadata', 'showing');
+      mockTextTracks(video, [metadataTrack]);
+
+      const store = createStore<PlayerTarget>()(textTrackFeature);
+      store.attach({ media: video, container: null });
+
+      expect(store.state.toggleSubtitles()).toBe(false);
+    });
+
     it('stops updating after destroy', () => {
       const video = createVideo();
       const store = createStore<PlayerTarget>()(textTrackFeature);

packages/core/src/dom/store/features/text-track.ts

@@ -1,14 +1,29 @@
-import { findTrackElement, listen } from '@videojs/utils/dom';
+import { findTrackElement, getSubtitlesTracks, listen } from '@videojs/utils/dom';
 
-import type { MediaTextCue, MediaTextTrackState } from '../../../core/media/state';
+import type { MediaTextCue, MediaTextTrack, MediaTextTrackState } from '../../../core/media/state';
 import { definePlayerFeature } from '../../feature';
 
 export const textTrackFeature = definePlayerFeature({
   name: 'textTrack',
-  state: (): MediaTextTrackState => ({
+  state: ({ target }): MediaTextTrackState => ({
     chaptersCues: [],
     thumbnailCues: [],
     thumbnailTrackSrc: null,
+    subtitlesList: [],
+    subtitlesShowing: false,
+    toggleSubtitles(forceShow?: boolean) {
+      const subtitlesTracks = getSubtitlesTracks(target().media);
+      if (!subtitlesTracks.length) return false;
+
+      const showing = subtitlesTracks.some((track: TextTrack) => track.mode === 'showing');
+      const nextShowing = forceShow ?? !showing;
+
+      for (const track of subtitlesTracks) {
+        track.mode = nextShowing ? 'showing' : 'disabled';
+      }
+
+      return nextShowing;
+    },
   }),
 
   attach({ target, signal, set }) {
@@ -22,11 +37,26 @@ export const textTrackFeature = definePlayerFeature({
 
       let chaptersTrack: TextTrack | null = null;
       let thumbnailTrack: TextTrack | null = null;
+      const subtitlesList: MediaTextTrack<'subtitles' | 'captions'>[] = [];
+      let subtitlesShowing = false;
 
       for (let i = 0; i < media.textTracks.length; i++) {
         const track = media.textTracks[i]!;
         if (!chaptersTrack && track.kind === 'chapters') chaptersTrack = track;
         if (!thumbnailTrack && track.kind === 'metadata' && track.label === 'thumbnails') thumbnailTrack = track;
+        if (track.kind === 'captions' || track.kind === 'subtitles') {
+          const showing = track.mode === 'showing';
+          subtitlesList.push({
+            kind: track.kind,
+            label: track.label,
+            language: track.language,
+            mode: track.mode,
+          });
+
+          if (showing) {
+            subtitlesShowing = true;
+          }
+        }
       }
 
       // VTTCue extends TextTrackCue with `text` — cast via `unknown` since
@@ -47,13 +77,13 @@ export const textTrackFeature = definePlayerFeature({
       // Listen for <track> load events on tracks that don't have cues yet.
       // `addtrack` fires before cues are parsed — we need the `load` event
       // on the <track> element to know when cues are ready.
-      for (const trackEl of media.querySelectorAll('track')) {
+      for (const trackEl of media.querySelectorAll?.('track') ?? []) {
         if (!trackEl.track?.cues?.length) {
           listen(trackEl, 'load', sync, { signal: trackCleanup.signal });
         }
       }
 
-      set({ chaptersCues, thumbnailCues, thumbnailTrackSrc });
+      set({ chaptersCues, thumbnailCues, thumbnailTrackSrc, subtitlesList, subtitlesShowing });
     }
 
     sync();