From 67b39fe4bd44a557ce45884b1d84b0a0d62592e3 Mon Sep 17 00:00:00 2001 From: Wesley Luyten Date: Sun, 14 Jun 2026 14:39:55 -0700 Subject: [PATCH] feat(core): add media tracks and renditions support (#1664) --- .../media-tracks/audio-rendition-list.ts | 142 ++++++++++ .../media/media-tracks/audio-rendition.ts | 24 ++ .../media/media-tracks/audio-track-list.ts | 128 +++++++++ .../core/media/media-tracks/audio-track.ts | 45 ++++ .../core/media/media-tracks/change-event.ts | 11 + .../src/core/media/media-tracks/global.ts | 19 ++ .../core/src/core/media/media-tracks/index.ts | 13 + .../core/src/core/media/media-tracks/mixin.ts | 249 ++++++++++++++++++ .../media/media-tracks/rendition-event.ts | 11 + .../media-tracks/tests/mixin-native.test.ts | 104 ++++++++ .../media/media-tracks/tests/mixin.test.ts | 85 ++++++ .../core/src/core/media/media-tracks/utils.ts | 12 + .../media-tracks/video-rendition-list.ts | 142 ++++++++++ .../media/media-tracks/video-rendition.ts | 27 ++ .../media/media-tracks/video-track-list.ts | 141 ++++++++++ .../core/media/media-tracks/video-track.ts | 50 ++++ packages/core/src/core/media/types.ts | 123 +++++++++ packages/core/src/dom/media/dash/index.ts | 3 +- packages/core/src/dom/media/hls/hlsjs.ts | 30 ++- packages/core/src/dom/media/hls/index.ts | 16 ++ .../core/src/dom/media/hls/media-tracks.ts | 177 +++++++++++++ .../dom/media/hls/tests/media-tracks.test.ts | 175 ++++++++++++ .../src/dom/store/features/remote-playback.ts | 2 +- .../features/tests/remote-playback.test.ts | 64 +++++ packages/html/src/store/media-attach-mixin.ts | 6 +- 25 files changed, 1792 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/core/media/media-tracks/audio-rendition-list.ts create mode 100644 packages/core/src/core/media/media-tracks/audio-rendition.ts create mode 100644 packages/core/src/core/media/media-tracks/audio-track-list.ts create mode 100644 packages/core/src/core/media/media-tracks/audio-track.ts create mode 100644 packages/core/src/core/media/media-tracks/change-event.ts create mode 100644 packages/core/src/core/media/media-tracks/global.ts create mode 100644 packages/core/src/core/media/media-tracks/index.ts create mode 100644 packages/core/src/core/media/media-tracks/mixin.ts create mode 100644 packages/core/src/core/media/media-tracks/rendition-event.ts create mode 100644 packages/core/src/core/media/media-tracks/tests/mixin-native.test.ts create mode 100644 packages/core/src/core/media/media-tracks/tests/mixin.test.ts create mode 100644 packages/core/src/core/media/media-tracks/utils.ts create mode 100644 packages/core/src/core/media/media-tracks/video-rendition-list.ts create mode 100644 packages/core/src/core/media/media-tracks/video-rendition.ts create mode 100644 packages/core/src/core/media/media-tracks/video-track-list.ts create mode 100644 packages/core/src/core/media/media-tracks/video-track.ts create mode 100644 packages/core/src/dom/media/hls/media-tracks.ts create mode 100644 packages/core/src/dom/media/hls/tests/media-tracks.test.ts create mode 100644 packages/core/src/dom/store/features/tests/remote-playback.test.ts diff --git a/packages/core/src/core/media/media-tracks/audio-rendition-list.ts b/packages/core/src/core/media/media-tracks/audio-rendition-list.ts new file mode 100644 index 00000000..ba7d3597 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/audio-rendition-list.ts @@ -0,0 +1,142 @@ +import { isFunction } from '@videojs/utils/predicate'; +import type { AudioRendition } from './audio-rendition'; +import type { AudioTrack } from './audio-track'; +import { RenditionEvent } from './rendition-event'; +import { getPrivate } from './utils'; + +export function addRendition(track: AudioTrack, rendition: AudioRendition) { + const renditionList = getPrivate(track).media?.deref()?.audioRenditions as AudioRenditionList | undefined; + + getPrivate(rendition).media = getPrivate(track).media; + getPrivate(rendition).track = track; + + const renditionSet = getPrivate(track).renditionSet as Set; + renditionSet.add(rendition); + const index = renditionSet.size - 1; + + if (!(index in AudioRenditionList.prototype)) { + Object.defineProperty(AudioRenditionList.prototype, index, { + get() { + return getCurrentRenditions(this)[index]; + }, + }); + } + + queueMicrotask(() => { + if (!renditionList || !track.enabled) return; + + renditionList.dispatchEvent(new RenditionEvent('addrendition', { rendition })); + }); +} + +export function removeRendition(rendition: AudioRendition) { + const renditionList = getPrivate(rendition).media?.deref()?.audioRenditions as AudioRenditionList | undefined; + const track = getPrivate(rendition).track as AudioTrack; + const renditionSet = getPrivate(track).renditionSet as Set; + renditionSet.delete(rendition); + + queueMicrotask(() => { + if (!renditionList || !track.enabled) return; + + renditionList.dispatchEvent(new RenditionEvent('removerendition', { rendition })); + }); +} + +export function selectedChanged(rendition: AudioRendition) { + const renditionList = getPrivate(rendition).media?.deref()?.audioRenditions as AudioRenditionList | undefined; + + // Prevent firing a rendition list `change` event multiple times per tick. + if (!renditionList || getPrivate(renditionList).changeRequested) return; + getPrivate(renditionList).changeRequested = true; + + queueMicrotask(() => { + delete getPrivate(renditionList).changeRequested; + + const track = getPrivate(rendition).track as AudioTrack; + if (!track.enabled) return; + + renditionList.dispatchEvent(new Event('change')); + }); +} + +function getCurrentRenditions(renditionList: AudioRenditionList): AudioRendition[] { + const media = getPrivate(renditionList).media?.deref() as HTMLMediaElement | undefined; + if (!media) return []; + return [...media.audioTracks] + .filter((track) => track.enabled) + .flatMap((track) => [...(getPrivate(track).renditionSet as Set)]); +} + +export class AudioRenditionList extends EventTarget { + [index: number]: AudioRendition; + #addRenditionCallback: (() => void) | undefined; + #removeRenditionCallback: (() => void) | undefined; + #changeCallback: (() => void) | undefined; + + [Symbol.iterator]() { + return getCurrentRenditions(this).values(); + } + + get length() { + return getCurrentRenditions(this).length; + } + + getRenditionById(id: string): AudioRendition | null { + return getCurrentRenditions(this).find((rendition) => `${rendition.id}` === `${id}`) ?? null; + } + + get selectedIndex() { + return getCurrentRenditions(this).findIndex((rendition) => rendition.selected); + } + + set selectedIndex(index) { + for (const [i, rendition] of getCurrentRenditions(this).entries()) { + rendition.selected = i === index; + } + } + + get onaddrendition() { + return this.#addRenditionCallback; + } + + set onaddrendition(callback: ((event?: { rendition: AudioRendition }) => void) | undefined) { + if (this.#addRenditionCallback) { + this.removeEventListener('addrendition', this.#addRenditionCallback); + this.#addRenditionCallback = undefined; + } + if (isFunction(callback)) { + this.#addRenditionCallback = callback; + this.addEventListener('addrendition', callback as unknown as EventListener); + } + } + + get onremoverendition() { + return this.#removeRenditionCallback; + } + + set onremoverendition(callback: ((event?: { rendition: AudioRendition }) => void) | undefined) { + if (this.#removeRenditionCallback) { + this.removeEventListener('removerendition', this.#removeRenditionCallback); + this.#removeRenditionCallback = undefined; + } + if (isFunction(callback)) { + this.#removeRenditionCallback = callback; + this.addEventListener('removerendition', callback as unknown as EventListener); + } + } + + get onchange() { + return this.#changeCallback; + } + + set onchange(callback) { + if (this.#changeCallback) { + this.removeEventListener('change', this.#changeCallback); + this.#changeCallback = undefined; + } + if (isFunction(callback)) { + this.#changeCallback = callback; + this.addEventListener('change', callback); + } + } +} diff --git a/packages/core/src/core/media/media-tracks/audio-rendition.ts b/packages/core/src/core/media/media-tracks/audio-rendition.ts new file mode 100644 index 00000000..6e3102c1 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/audio-rendition.ts @@ -0,0 +1,24 @@ +import { selectedChanged } from './audio-rendition-list'; + +/** + * The consumer should use the `selected` setter to select one or multiple + * renditions that the engine is allowed to play. + */ +export class AudioRendition { + src: string | undefined; + id: string | undefined; + bitrate: number | undefined; + codec: string | undefined; + #selected = false; + + get selected(): boolean { + return this.#selected; + } + + set selected(value: boolean) { + if (this.#selected === value) return; + this.#selected = value; + + selectedChanged(this); + } +} diff --git a/packages/core/src/core/media/media-tracks/audio-track-list.ts b/packages/core/src/core/media/media-tracks/audio-track-list.ts new file mode 100644 index 00000000..9e6ecce5 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/audio-track-list.ts @@ -0,0 +1,128 @@ +import { isFunction } from '@videojs/utils/predicate'; +import type { AudioTrack } from './audio-track'; +import { TrackEvent } from './change-event'; +import { getPrivate } from './utils'; + +export function addAudioTrack(media: HTMLMediaElement, track: AudioTrack) { + const trackList = media.audioTracks; + getPrivate(track).media = new WeakRef(media); + + if (!getPrivate(track).renditionSet) { + getPrivate(track).renditionSet = new Set(); + } + + const trackSet = getPrivate(trackList).trackSet as Set; + trackSet.add(track); + const index = trackSet.size - 1; + + if (!(index in AudioTrackList.prototype)) { + Object.defineProperty(AudioTrackList.prototype, index, { + get() { + return [...(getPrivate(this).trackSet as Set)][index]; + }, + }); + } + + // The event is queued to align with native `addtrack` behavior. + queueMicrotask(() => { + trackList.dispatchEvent(new TrackEvent('addtrack', { track })); + }); +} + +export function removeAudioTrack(track: AudioTrack) { + const trackList = getPrivate(track).media?.deref()?.audioTracks as AudioTrackList | undefined; + if (!trackList) return; + + const trackSet = getPrivate(trackList).trackSet as Set; + if (!trackSet.delete(track)) return; + + queueMicrotask(() => { + trackList.dispatchEvent(new TrackEvent('removetrack', { track })); + }); +} + +export function enabledChanged(track: AudioTrack) { + const trackList = getPrivate(track).media?.deref()?.audioTracks as AudioTrackList | undefined; + + // Prevent firing a track list `change` event multiple times per tick. + if (!trackList || getPrivate(trackList).changeRequested) return; + getPrivate(trackList).changeRequested = true; + + queueMicrotask(() => { + delete getPrivate(trackList).changeRequested; + trackList.dispatchEvent(new Event('change')); + }); +} + +export class AudioTrackList extends EventTarget { + [index: number]: AudioTrack; + #addTrackCallback: (() => void) | undefined; + #removeTrackCallback: (() => void) | undefined; + #changeCallback: (() => void) | undefined; + + constructor() { + super(); + getPrivate(this).trackSet = new Set(); + } + + get #tracks() { + return getPrivate(this).trackSet as Set; + } + + [Symbol.iterator]() { + return this.#tracks.values(); + } + + get length() { + return this.#tracks.size; + } + + getTrackById(id: string): AudioTrack | null { + return [...this.#tracks].find((track) => track.id === id) ?? null; + } + + get onaddtrack() { + return this.#addTrackCallback; + } + + set onaddtrack(callback: ((event?: { track: AudioTrack }) => void) | undefined) { + if (this.#addTrackCallback) { + this.removeEventListener('addtrack', this.#addTrackCallback); + this.#addTrackCallback = undefined; + } + if (isFunction(callback)) { + this.#addTrackCallback = callback; + this.addEventListener('addtrack', callback as unknown as EventListener); + } + } + + get onremovetrack() { + return this.#removeTrackCallback; + } + + set onremovetrack(callback: ((event?: { track: AudioTrack }) => void) | undefined) { + if (this.#removeTrackCallback) { + this.removeEventListener('removetrack', this.#removeTrackCallback); + this.#removeTrackCallback = undefined; + } + if (isFunction(callback)) { + this.#removeTrackCallback = callback; + this.addEventListener('removetrack', callback as unknown as EventListener); + } + } + + get onchange() { + return this.#changeCallback; + } + + set onchange(callback) { + if (this.#changeCallback) { + this.removeEventListener('change', this.#changeCallback); + this.#changeCallback = undefined; + } + if (isFunction(callback)) { + this.#changeCallback = callback; + this.addEventListener('change', callback); + } + } +} diff --git a/packages/core/src/core/media/media-tracks/audio-track.ts b/packages/core/src/core/media/media-tracks/audio-track.ts new file mode 100644 index 00000000..a2a2607e --- /dev/null +++ b/packages/core/src/core/media/media-tracks/audio-track.ts @@ -0,0 +1,45 @@ +import { AudioRendition } from './audio-rendition'; +import { addRendition, removeRendition } from './audio-rendition-list'; +import { enabledChanged } from './audio-track-list'; + +export const AudioTrackKind = { + alternative: 'alternative', + descriptions: 'descriptions', + main: 'main', + 'main-desc': 'main-desc', + translation: 'translation', + commentary: 'commentary', +}; + +export class AudioTrack { + id: string | undefined; + kind: string | undefined; + label = ''; + language = ''; + sourceBuffer: unknown; + #enabled = false; + + addRendition(src: string, codec?: string, bitrate?: number) { + const rendition = new AudioRendition(); + rendition.src = src; + rendition.codec = codec; + rendition.bitrate = bitrate; + addRendition(this, rendition); + return rendition; + } + + removeRendition(rendition: AudioRendition) { + removeRendition(rendition); + } + + get enabled(): boolean { + return this.#enabled; + } + + set enabled(value: boolean) { + if (this.#enabled === value) return; + this.#enabled = value; + + enabledChanged(this); + } +} diff --git a/packages/core/src/core/media/media-tracks/change-event.ts b/packages/core/src/core/media/media-tracks/change-event.ts new file mode 100644 index 00000000..c54913d5 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/change-event.ts @@ -0,0 +1,11 @@ +import type { AudioTrack } from './audio-track'; +import type { VideoTrack } from './video-track'; + +export class TrackEvent extends Event { + track: AudioTrack | VideoTrack; + + constructor(type: string, init: { track: AudioTrack | VideoTrack }) { + super(type); + this.track = init.track; + } +} diff --git a/packages/core/src/core/media/media-tracks/global.ts b/packages/core/src/core/media/media-tracks/global.ts new file mode 100644 index 00000000..05f2c411 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/global.ts @@ -0,0 +1,19 @@ +import type { AudioRenditionList } from './audio-rendition-list'; +import type { AudioTrack } from './audio-track'; +import type { AudioTrackList } from './audio-track-list'; +import type { VideoRenditionList } from './video-rendition-list'; +import type { VideoTrack } from './video-track'; +import type { VideoTrackList } from './video-track-list'; + +declare global { + interface HTMLMediaElement { + videoTracks: VideoTrackList; + audioTracks: AudioTrackList; + addVideoTrack(kind: string, label?: string, language?: string): VideoTrack; + addAudioTrack(kind: string, label?: string, language?: string): AudioTrack; + removeVideoTrack(track: VideoTrack): void; + removeAudioTrack(track: AudioTrack): void; + videoRenditions: VideoRenditionList; + audioRenditions: AudioRenditionList; + } +} diff --git a/packages/core/src/core/media/media-tracks/index.ts b/packages/core/src/core/media/media-tracks/index.ts new file mode 100644 index 00000000..bde51d97 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/index.ts @@ -0,0 +1,13 @@ +import './global'; + +export { AudioRendition } from './audio-rendition'; +export { AudioRenditionList } from './audio-rendition-list'; +export { AudioTrack, AudioTrackKind } from './audio-track'; +export { AudioTrackList } from './audio-track-list'; +export { TrackEvent } from './change-event'; +export * from './mixin'; +export { RenditionEvent } from './rendition-event'; +export { VideoRendition } from './video-rendition'; +export { VideoRenditionList } from './video-rendition-list'; +export { VideoTrack, VideoTrackKind } from './video-track'; +export { VideoTrackList } from './video-track-list'; diff --git a/packages/core/src/core/media/media-tracks/mixin.ts b/packages/core/src/core/media/media-tracks/mixin.ts new file mode 100644 index 00000000..55c3fd5b --- /dev/null +++ b/packages/core/src/core/media/media-tracks/mixin.ts @@ -0,0 +1,249 @@ +import type { AnyConstructor, MixinReturn } from '@videojs/utils/types'; +import type { + MediaAudioRenditionCapability, + MediaAudioTrackCapability, + MediaVideoRenditionCapability, + MediaVideoTrackCapability, +} from '../types'; +import { AudioRenditionList } from './audio-rendition-list'; +import { AudioTrack } from './audio-track'; +import { AudioTrackList, addAudioTrack, removeAudioTrack } from './audio-track-list'; +import type { TrackEvent } from './change-event'; +import './global'; +import { getPrivate } from './utils'; +import { VideoRenditionList } from './video-rendition-list'; +import { VideoTrack } from './video-track'; +import { addVideoTrack, removeVideoTrack, VideoTrackList } from './video-track-list'; + +export type WithMediaTracks> = MixinReturn< + Base, + MediaVideoTrackCapability & MediaAudioTrackCapability & MediaVideoRenditionCapability & MediaAudioRenditionCapability +>; + +const HTMLMediaElementConstructor = (globalThis as { HTMLMediaElement?: AnyConstructor }) + .HTMLMediaElement; +const nativeVideoTracksFn = getBaseMediaTracksFn(HTMLMediaElementConstructor, 'video'); +const nativeAudioTracksFn = getBaseMediaTracksFn(HTMLMediaElementConstructor, 'audio'); + +// Safari supports native media tracks, but native implementations cannot +// reliably represent manifest-derived MSE tracks or manually-added tracks. +export function MediaTracksMixin>(MediaElementClass: Base): WithMediaTracks { + if (!MediaElementClass?.prototype) return MediaElementClass as WithMediaTracks; + + const prototype = MediaElementClass.prototype as Record; + const videoTracksFn = getBaseMediaTracksFn(MediaElementClass, 'video'); + + if (!videoTracksFn || `${videoTracksFn}`.includes('[native code]')) { + Object.defineProperty(prototype, 'videoTracks', { + get() { + return getVideoTracks(this); + }, + }); + } + + const audioTracksFn = getBaseMediaTracksFn(MediaElementClass, 'audio'); + + if (!audioTracksFn || `${audioTracksFn}`.includes('[native code]')) { + Object.defineProperty(prototype, 'audioTracks', { + get() { + return getAudioTracks(this); + }, + }); + } + + if (!hasOwn(prototype, 'addVideoTrack')) { + prototype.addVideoTrack = function (this: HTMLMediaElement, kind: string, label = '', language = '') { + const track = new VideoTrack(); + track.kind = kind; + track.label = label; + track.language = language; + addVideoTrack(this, track); + return track; + }; + } + + if (!hasOwn(prototype, 'removeVideoTrack')) { + prototype.removeVideoTrack = removeVideoTrack; + } + + if (!hasOwn(prototype, 'addAudioTrack')) { + prototype.addAudioTrack = function (this: HTMLMediaElement, kind: string, label = '', language = '') { + const track = new AudioTrack(); + track.kind = kind; + track.label = label; + track.language = language; + addAudioTrack(this, track); + return track; + }; + } + + if (!hasOwn(prototype, 'removeAudioTrack')) { + prototype.removeAudioTrack = removeAudioTrack; + } + + // Tear down the native-track listeners wired in getVideoTracks/getAudioTracks + // when the target is removed, and drop the cached lists so a re-attach + // re-mirrors against the new target. + if (!hasOwn(prototype, 'detach')) { + const baseDetach = prototype.detach as ((this: object) => void) | undefined; + prototype.detach = function (this: object) { + const priv = getPrivate(this); + (priv.videoTracksCleanup as AbortController | undefined)?.abort(); + (priv.audioTracksCleanup as AbortController | undefined)?.abort(); + delete priv.videoTracks; + delete priv.audioTracks; + delete priv.videoTracksCleanup; + delete priv.audioTracksCleanup; + baseDetach?.call(this); + }; + } + + if (!hasOwn(prototype, 'videoRenditions')) { + Object.defineProperty(prototype, 'videoRenditions', { + get() { + return initVideoRenditions(this); + }, + }); + } + + if (!hasOwn(prototype, 'audioRenditions')) { + Object.defineProperty(prototype, 'audioRenditions', { + get() { + return initAudioRenditions(this); + }, + }); + } + + return MediaElementClass as WithMediaTracks; +} + +function hasOwn(value: object, key: PropertyKey) { + return Object.hasOwn(value, key); +} + +function initVideoRenditions(media: HTMLMediaElement) { + let renditions = getPrivate(media).videoRenditions as VideoRenditionList | undefined; + if (!renditions) { + renditions = new VideoRenditionList(); + getPrivate(renditions).media = new WeakRef(media); + getPrivate(media).videoRenditions = renditions; + } + return renditions; +} + +function initAudioRenditions(media: HTMLMediaElement) { + let renditions = getPrivate(media).audioRenditions as AudioRenditionList | undefined; + if (!renditions) { + renditions = new AudioRenditionList(); + getPrivate(renditions).media = new WeakRef(media); + getPrivate(media).audioRenditions = renditions; + } + return renditions; +} + +function getBaseMediaTracksFn(MediaElementClass: any, type: string): (() => any) | undefined { + if (MediaElementClass?.prototype) { + return Object.getOwnPropertyDescriptor(MediaElementClass.prototype, `${type}Tracks`)?.get; + } + return undefined; +} + +function getVideoTracks(media: any) { + let tracks = getPrivate(media).videoTracks as VideoTrackList | undefined; + if (!tracks) { + tracks = new VideoTrackList(); + getPrivate(media).videoTracks = tracks; + + const nativeEl = media.target; + + if (nativeVideoTracksFn && nativeEl) { + const currentTracks = tracks; + const nativeTracks = nativeVideoTracksFn.call(nativeEl); + + for (const nativeTrack of nativeTracks) { + addVideoTrack(media, nativeTrack); + } + + const onChange = () => { + currentTracks.dispatchEvent(new Event('change')); + }; + + const onAddTrack = (event: TrackEvent) => { + if ([...currentTracks].some((track) => track instanceof VideoTrack)) return; + addVideoTrack(media, event.track as VideoTrack); + }; + + const onRemoveTrack = (event: TrackEvent) => { + removeVideoTrack(event.track as VideoTrack); + }; + + // Adding a custom track replaces any mirrored native tracks. + const onCustomAddTrack = (event: Event) => { + if (!((event as TrackEvent).track instanceof VideoTrack)) return; + for (const nativeTrack of nativeTracks) { + removeVideoTrack(nativeTrack as VideoTrack); + } + }; + + const controller = new AbortController(); + const { signal } = controller; + getPrivate(media).videoTracksCleanup = controller; + + nativeTracks.addEventListener('change', onChange, { signal }); + nativeTracks.addEventListener('addtrack', onAddTrack, { signal }); + nativeTracks.addEventListener('removetrack', onRemoveTrack, { signal }); + currentTracks.addEventListener('addtrack', onCustomAddTrack, { signal }); + } + } + return tracks; +} + +function getAudioTracks(media: any) { + let tracks = getPrivate(media).audioTracks as AudioTrackList | undefined; + if (!tracks) { + tracks = new AudioTrackList(); + getPrivate(media).audioTracks = tracks; + + const nativeEl = media.target; + + if (nativeAudioTracksFn && nativeEl) { + const currentTracks = tracks; + const nativeTracks = nativeAudioTracksFn.call(nativeEl); + + for (const nativeTrack of nativeTracks) { + addAudioTrack(media, nativeTrack); + } + + const onChange = () => { + currentTracks.dispatchEvent(new Event('change')); + }; + + const onAddTrack = (event: TrackEvent) => { + if ([...currentTracks].some((track) => track instanceof AudioTrack)) return; + addAudioTrack(media, event.track as AudioTrack); + }; + + const onRemoveTrack = (event: TrackEvent) => { + removeAudioTrack(event.track as AudioTrack); + }; + + // Adding a custom track replaces any mirrored native tracks. + const onCustomAddTrack = (event: Event) => { + if (!((event as TrackEvent).track instanceof AudioTrack)) return; + for (const nativeTrack of nativeTracks) { + removeAudioTrack(nativeTrack as AudioTrack); + } + }; + + const controller = new AbortController(); + const { signal } = controller; + getPrivate(media).audioTracksCleanup = controller; + + nativeTracks.addEventListener('change', onChange, { signal }); + nativeTracks.addEventListener('addtrack', onAddTrack, { signal }); + nativeTracks.addEventListener('removetrack', onRemoveTrack, { signal }); + currentTracks.addEventListener('addtrack', onCustomAddTrack, { signal }); + } + } + return tracks; +} diff --git a/packages/core/src/core/media/media-tracks/rendition-event.ts b/packages/core/src/core/media/media-tracks/rendition-event.ts new file mode 100644 index 00000000..cee9c031 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/rendition-event.ts @@ -0,0 +1,11 @@ +import type { AudioRendition } from './audio-rendition'; +import type { VideoRendition } from './video-rendition'; + +export class RenditionEvent extends Event { + rendition: AudioRendition | VideoRendition; + + constructor(type: string, init: { rendition: AudioRendition | VideoRendition }) { + super(type); + this.rendition = init.rendition; + } +} diff --git a/packages/core/src/core/media/media-tracks/tests/mixin-native.test.ts b/packages/core/src/core/media/media-tracks/tests/mixin-native.test.ts new file mode 100644 index 00000000..76ae7902 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/tests/mixin-native.test.ts @@ -0,0 +1,104 @@ +import { afterAll, describe, expect, it, vi } from 'vitest'; + +class FakeNativeTrackList extends EventTarget { + #tracks: object[] = []; + + [Symbol.iterator]() { + return this.#tracks.values(); + } + + add(track: object) { + this.#tracks.push(track); + const event = new Event('addtrack') as Event & { track: object }; + event.track = track; + this.dispatchEvent(event); + } +} + +class FakeHTMLMediaElement { + #videoTracks = new FakeNativeTrackList(); + #audioTracks = new FakeNativeTrackList(); + + get videoTracks() { + return this.#videoTracks; + } + + get audioTracks() { + return this.#audioTracks; + } +} + +// The mixin captures the native track getters from `globalThis.HTMLMediaElement` +// at module load, so stub the global before importing it. +vi.stubGlobal('HTMLMediaElement', FakeHTMLMediaElement); +vi.resetModules(); + +const { MediaTracksMixin } = await import('../mixin'); + +afterAll(() => { + vi.unstubAllGlobals(); +}); + +class NativeMedia extends EventTarget { + target = new FakeHTMLMediaElement(); +} + +const NativeMediaWithTracks = MediaTracksMixin(NativeMedia); + +describe('MediaTracksMixin', () => { + it('mirrors native tracks into the custom lists', () => { + const media = new NativeMediaWithTracks(); + const nativeVideoTrack = { kind: 'main' }; + + media.target.videoTracks.add(nativeVideoTrack); + + expect(media.videoTracks.length).toBe(1); + expect([...media.videoTracks]).toEqual([nativeVideoTrack]); + }); + + it('removes mirrored native video tracks when a custom track is added', async () => { + const media = new NativeMediaWithTracks(); + + media.target.videoTracks.add({ kind: 'main' }); + expect(media.videoTracks.length).toBe(1); + + const custom = media.addVideoTrack('main'); + await Promise.resolve(); + + expect(media.videoTracks.length).toBe(1); + expect([...media.videoTracks]).toEqual([custom]); + }); + + it('removes mirrored native audio tracks when a custom track is added', async () => { + const media = new NativeMediaWithTracks(); + + media.target.audioTracks.add({ kind: 'main' }); + expect(media.audioTracks.length).toBe(1); + + const custom = media.addAudioTrack('main'); + await Promise.resolve(); + + expect(media.audioTracks.length).toBe(1); + expect([...media.audioTracks]).toEqual([custom]); + }); + + it('removes native track listeners when the target is detached', () => { + const media = new NativeMediaWithTracks(); + const videoList = media.videoTracks; + const audioList = media.audioTracks; + + media.target.videoTracks.add({ kind: 'main' }); + media.target.audioTracks.add({ kind: 'main' }); + expect(videoList.length).toBe(1); + expect(audioList.length).toBe(1); + + (media as unknown as { detach(): void }).detach(); + + media.target.videoTracks.add({ kind: 'alternative' }); + media.target.audioTracks.add({ kind: 'alternative' }); + + // Listeners were torn down, so the detached lists no longer mirror. + expect(videoList.length).toBe(1); + expect(audioList.length).toBe(1); + }); +}); diff --git a/packages/core/src/core/media/media-tracks/tests/mixin.test.ts b/packages/core/src/core/media/media-tracks/tests/mixin.test.ts new file mode 100644 index 00000000..9512baf1 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/tests/mixin.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it, vi } from 'vitest'; +import { MediaTracksMixin } from '../mixin'; + +class TestMedia extends EventTarget {} +const TestMediaWithTracks = MediaTracksMixin(TestMedia); + +describe('MediaTracksMixin', () => { + it('backs audioTracks with track-list authoring methods', () => { + const media = new TestMediaWithTracks(); + + const track = media.addAudioTrack('main', 'English', 'en'); + + expect(media.audioTracks.length).toBe(1); + expect(media.audioTracks[0]).toBe(track); + expect([...media.audioTracks]).toEqual([track]); + expect(track.label).toBe('English'); + expect(track.language).toBe('en'); + }); + + it('removes audio tracks', () => { + const media = new TestMediaWithTracks(); + + const track = media.addAudioTrack('main'); + expect(media.audioTracks.length).toBe(1); + + media.removeAudioTrack(track); + expect(media.audioTracks.length).toBe(0); + }); + + it('exposes renditions of the selected video track only', () => { + const media = new TestMediaWithTracks(); + + const selected = media.addVideoTrack('main'); + selected.selected = true; + selected.addRendition('high.m3u8', 1920, 1080); + selected.addRendition('low.m3u8', 640, 360); + + const hidden = media.addVideoTrack('alternative'); + hidden.addRendition('alt.m3u8', 1280, 720); + + expect(media.videoTracks.length).toBe(2); + expect(media.videoRenditions.length).toBe(2); + expect([...media.videoRenditions].map((rendition) => rendition.width)).toEqual([1920, 640]); + }); + + it('does not dispatch removetrack when the video track is not in the list', async () => { + const media = new TestMediaWithTracks(); + const onRemoveTrack = vi.fn(); + media.videoTracks.addEventListener('removetrack', onRemoveTrack); + + const track = media.addVideoTrack('main'); + media.removeVideoTrack(track); + media.removeVideoTrack(track); + await Promise.resolve(); + + expect(onRemoveTrack).toHaveBeenCalledTimes(1); + }); + + it('does not dispatch removetrack when the audio track is not in the list', async () => { + const media = new TestMediaWithTracks(); + const onRemoveTrack = vi.fn(); + media.audioTracks.addEventListener('removetrack', onRemoveTrack); + + const track = media.addAudioTrack('main'); + media.removeAudioTrack(track); + media.removeAudioTrack(track); + await Promise.resolve(); + + expect(onRemoveTrack).toHaveBeenCalledTimes(1); + }); + + it('makes video track selection exclusive', () => { + const media = new TestMediaWithTracks(); + + const first = media.addVideoTrack('main'); + const second = media.addVideoTrack('alternative'); + + first.selected = true; + second.selected = true; + + expect(first.selected).toBe(false); + expect(second.selected).toBe(true); + expect(media.videoTracks.selectedIndex).toBe(1); + }); +}); diff --git a/packages/core/src/core/media/media-tracks/utils.ts b/packages/core/src/core/media/media-tracks/utils.ts new file mode 100644 index 00000000..1899a5cd --- /dev/null +++ b/packages/core/src/core/media/media-tracks/utils.ts @@ -0,0 +1,12 @@ +const privateProps = new WeakMap>(); + +export function getPrivate(instance: object) { + return privateProps.get(instance) ?? setPrivate(instance, {}); +} + +export function setPrivate(instance: object, props: Record) { + let saved = privateProps.get(instance); + if (!saved) privateProps.set(instance, (saved = {})); + + return Object.assign(saved, props); +} diff --git a/packages/core/src/core/media/media-tracks/video-rendition-list.ts b/packages/core/src/core/media/media-tracks/video-rendition-list.ts new file mode 100644 index 00000000..3d860cfa --- /dev/null +++ b/packages/core/src/core/media/media-tracks/video-rendition-list.ts @@ -0,0 +1,142 @@ +import { isFunction } from '@videojs/utils/predicate'; +import { RenditionEvent } from './rendition-event'; +import { getPrivate } from './utils'; +import type { VideoRendition } from './video-rendition'; +import type { VideoTrack } from './video-track'; + +export function addRendition(track: VideoTrack, rendition: VideoRendition) { + const renditionList = getPrivate(track).media?.deref()?.videoRenditions as VideoRenditionList | undefined; + + getPrivate(rendition).media = getPrivate(track).media; + getPrivate(rendition).track = track; + + const renditionSet = getPrivate(track).renditionSet as Set; + renditionSet.add(rendition); + const index = renditionSet.size - 1; + + if (!(index in VideoRenditionList.prototype)) { + Object.defineProperty(VideoRenditionList.prototype, index, { + get() { + return getCurrentRenditions(this)[index]; + }, + }); + } + + queueMicrotask(() => { + if (!renditionList || !track.selected) return; + + renditionList.dispatchEvent(new RenditionEvent('addrendition', { rendition })); + }); +} + +export function removeRendition(rendition: VideoRendition) { + const renditionList = getPrivate(rendition).media?.deref()?.videoRenditions as VideoRenditionList | undefined; + const track = getPrivate(rendition).track as VideoTrack; + const renditionSet = getPrivate(track).renditionSet as Set; + renditionSet.delete(rendition); + + queueMicrotask(() => { + if (!renditionList || !track.selected) return; + + renditionList.dispatchEvent(new RenditionEvent('removerendition', { rendition })); + }); +} + +export function selectedChanged(rendition: VideoRendition) { + const renditionList = getPrivate(rendition).media?.deref()?.videoRenditions as VideoRenditionList | undefined; + + // Prevent firing a rendition list `change` event multiple times per tick. + if (!renditionList || getPrivate(renditionList).changeRequested) return; + getPrivate(renditionList).changeRequested = true; + + queueMicrotask(() => { + delete getPrivate(renditionList).changeRequested; + + const track = getPrivate(rendition).track as VideoTrack; + if (!track.selected) return; + + renditionList.dispatchEvent(new Event('change')); + }); +} + +function getCurrentRenditions(renditionList: VideoRenditionList): VideoRendition[] { + const media = getPrivate(renditionList).media?.deref() as HTMLMediaElement | undefined; + if (!media) return []; + return [...media.videoTracks] + .filter((track) => track.selected) + .flatMap((track) => [...(getPrivate(track).renditionSet as Set)]); +} + +export class VideoRenditionList extends EventTarget { + [index: number]: VideoRendition; + #addRenditionCallback: (() => void) | undefined; + #removeRenditionCallback: (() => void) | undefined; + #changeCallback: (() => void) | undefined; + + [Symbol.iterator]() { + return getCurrentRenditions(this).values(); + } + + get length() { + return getCurrentRenditions(this).length; + } + + getRenditionById(id: string): VideoRendition | null { + return getCurrentRenditions(this).find((rendition) => `${rendition.id}` === `${id}`) ?? null; + } + + get selectedIndex() { + return getCurrentRenditions(this).findIndex((rendition) => rendition.selected); + } + + set selectedIndex(index) { + for (const [i, rendition] of getCurrentRenditions(this).entries()) { + rendition.selected = i === index; + } + } + + get onaddrendition() { + return this.#addRenditionCallback; + } + + set onaddrendition(callback: ((event?: { rendition: VideoRendition }) => void) | undefined) { + if (this.#addRenditionCallback) { + this.removeEventListener('addrendition', this.#addRenditionCallback); + this.#addRenditionCallback = undefined; + } + if (isFunction(callback)) { + this.#addRenditionCallback = callback; + this.addEventListener('addrendition', callback as unknown as EventListener); + } + } + + get onremoverendition() { + return this.#removeRenditionCallback; + } + + set onremoverendition(callback: ((event?: { rendition: VideoRendition }) => void) | undefined) { + if (this.#removeRenditionCallback) { + this.removeEventListener('removerendition', this.#removeRenditionCallback); + this.#removeRenditionCallback = undefined; + } + if (isFunction(callback)) { + this.#removeRenditionCallback = callback; + this.addEventListener('removerendition', callback as unknown as EventListener); + } + } + + get onchange() { + return this.#changeCallback; + } + + set onchange(callback) { + if (this.#changeCallback) { + this.removeEventListener('change', this.#changeCallback); + this.#changeCallback = undefined; + } + if (isFunction(callback)) { + this.#changeCallback = callback; + this.addEventListener('change', callback); + } + } +} diff --git a/packages/core/src/core/media/media-tracks/video-rendition.ts b/packages/core/src/core/media/media-tracks/video-rendition.ts new file mode 100644 index 00000000..81ca3ac7 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/video-rendition.ts @@ -0,0 +1,27 @@ +import { selectedChanged } from './video-rendition-list'; + +/** + * The consumer should use the `selected` setter to select one or multiple + * renditions that the engine is allowed to play. + */ +export class VideoRendition { + src: string | undefined; + id: string | undefined; + width: number | undefined; + height: number | undefined; + bitrate: number | undefined; + frameRate: number | undefined; + codec: string | undefined; + #selected = false; + + get selected(): boolean { + return this.#selected; + } + + set selected(value: boolean) { + if (this.#selected === value) return; + this.#selected = value; + + selectedChanged(this); + } +} diff --git a/packages/core/src/core/media/media-tracks/video-track-list.ts b/packages/core/src/core/media/media-tracks/video-track-list.ts new file mode 100644 index 00000000..6c8db4c9 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/video-track-list.ts @@ -0,0 +1,141 @@ +import { isFunction } from '@videojs/utils/predicate'; +import { TrackEvent } from './change-event'; +import { getPrivate } from './utils'; +import type { VideoTrack } from './video-track'; + +export function addVideoTrack(media: HTMLMediaElement, track: VideoTrack) { + const trackList = media.videoTracks; + getPrivate(track).media = new WeakRef(media); + + if (!getPrivate(track).renditionSet) { + getPrivate(track).renditionSet = new Set(); + } + + const trackSet = getPrivate(trackList).trackSet as Set; + trackSet.add(track); + const index = trackSet.size - 1; + + if (!(index in VideoTrackList.prototype)) { + Object.defineProperty(VideoTrackList.prototype, index, { + get() { + return [...(getPrivate(this).trackSet as Set)][index]; + }, + }); + } + + // The event is queued to align with native `addtrack` behavior. + queueMicrotask(() => { + trackList.dispatchEvent(new TrackEvent('addtrack', { track })); + }); +} + +export function removeVideoTrack(track: VideoTrack) { + const trackList = getPrivate(track).media?.deref()?.videoTracks as VideoTrackList | undefined; + if (!trackList) return; + + const trackSet = getPrivate(trackList).trackSet as Set; + if (!trackSet.delete(track)) return; + + queueMicrotask(() => { + trackList.dispatchEvent(new TrackEvent('removetrack', { track })); + }); +} + +export function selectedChanged(selected: VideoTrack) { + const trackList = (getPrivate(selected).media?.deref()?.videoTracks ?? []) as VideoTrackList | VideoTrack[]; + let hasUnselected = false; + + for (const track of trackList) { + if (track === selected) continue; + track.selected = false; + hasUnselected = true; + } + + if (!hasUnselected) return; + + // Prevent firing a track list `change` event multiple times per tick. + if (getPrivate(trackList).changeRequested) return; + getPrivate(trackList).changeRequested = true; + + queueMicrotask(() => { + delete getPrivate(trackList).changeRequested; + (trackList as VideoTrackList).dispatchEvent(new Event('change')); + }); +} + +export class VideoTrackList extends EventTarget { + [index: number]: VideoTrack; + #addTrackCallback: (() => void) | undefined; + #removeTrackCallback: (() => void) | undefined; + #changeCallback: (() => void) | undefined; + + constructor() { + super(); + getPrivate(this).trackSet = new Set(); + } + + get #tracks() { + return getPrivate(this).trackSet as Set; + } + + [Symbol.iterator]() { + return this.#tracks.values(); + } + + get length() { + return this.#tracks.size; + } + + getTrackById(id: string): VideoTrack | null { + return [...this.#tracks].find((track) => track.id === id) ?? null; + } + + get selectedIndex() { + return [...this.#tracks].findIndex((track) => track.selected); + } + + get onaddtrack() { + return this.#addTrackCallback; + } + + set onaddtrack(callback: ((event?: { track: VideoTrack }) => void) | undefined) { + if (this.#addTrackCallback) { + this.removeEventListener('addtrack', this.#addTrackCallback); + this.#addTrackCallback = undefined; + } + if (isFunction(callback)) { + this.#addTrackCallback = callback; + this.addEventListener('addtrack', callback as unknown as EventListener); + } + } + + get onremovetrack() { + return this.#removeTrackCallback; + } + + set onremovetrack(callback: ((event?: { track: VideoTrack }) => void) | undefined) { + if (this.#removeTrackCallback) { + this.removeEventListener('removetrack', this.#removeTrackCallback); + this.#removeTrackCallback = undefined; + } + if (isFunction(callback)) { + this.#removeTrackCallback = callback; + this.addEventListener('removetrack', callback as unknown as EventListener); + } + } + + get onchange() { + return this.#changeCallback; + } + + set onchange(callback) { + if (this.#changeCallback) { + this.removeEventListener('change', this.#changeCallback); + this.#changeCallback = undefined; + } + if (isFunction(callback)) { + this.#changeCallback = callback; + this.addEventListener('change', callback); + } + } +} diff --git a/packages/core/src/core/media/media-tracks/video-track.ts b/packages/core/src/core/media/media-tracks/video-track.ts new file mode 100644 index 00000000..5e615254 --- /dev/null +++ b/packages/core/src/core/media/media-tracks/video-track.ts @@ -0,0 +1,50 @@ +import { VideoRendition } from './video-rendition'; +import { addRendition, removeRendition } from './video-rendition-list'; +import { selectedChanged } from './video-track-list'; + +export const VideoTrackKind = { + alternative: 'alternative', + captions: 'captions', + main: 'main', + sign: 'sign', + subtitles: 'subtitles', + commentary: 'commentary', +}; + +export class VideoTrack { + id: string | undefined; + kind: string | undefined; + label = ''; + language = ''; + sourceBuffer: unknown; + #selected = false; + + addRendition(src: string, width?: number, height?: number, codec?: string, bitrate?: number, frameRate?: number) { + const rendition = new VideoRendition(); + rendition.src = src; + rendition.width = width; + rendition.height = height; + rendition.frameRate = frameRate; + rendition.bitrate = bitrate; + rendition.codec = codec; + addRendition(this, rendition); + return rendition; + } + + removeRendition(rendition: VideoRendition) { + removeRendition(rendition); + } + + get selected(): boolean { + return this.#selected; + } + + set selected(value: boolean) { + if (this.#selected === value) return; + this.#selected = value; + + if (value !== true) return; + + selectedChanged(this); + } +} diff --git a/packages/core/src/core/media/types.ts b/packages/core/src/core/media/types.ts index 2ce08090..dff551d5 100644 --- a/packages/core/src/core/media/types.ts +++ b/packages/core/src/core/media/types.ts @@ -255,6 +255,129 @@ export interface MediaTextTrackCapability { addTextTrack(kind: TextTrackKind, label?: string, language?: string): TextTrackLike; } +// ---------------------------------------- +// Media tracks +// ---------------------------------------- + +interface MediaTrackEventLike extends EventLike { + readonly track: Track; +} + +interface MediaTrackListEvents { + addtrack: MediaTrackEventLike; + removetrack: MediaTrackEventLike; + change: EventLike; +} + +export interface AudioTrackLike { + id: string | undefined; + readonly kind: string | undefined; + readonly label: string; + readonly language: string; + enabled: boolean; + addRendition(src: string, codec?: string | undefined, bitrate?: number | undefined): AudioRenditionLike; + removeRendition(rendition: AudioRenditionLike): void; +} + +export interface AudioTrackListLike extends EventTargetLike> { + readonly length: number; + readonly [index: number]: AudioTrackLike; + [Symbol.iterator](): Iterator; + getTrackById(id: string): AudioTrackLike | null; +} + +export interface VideoTrackLike { + id: string | undefined; + readonly kind: string | undefined; + readonly label: string; + readonly language: string; + selected: boolean; + addRendition( + src: string, + width?: number | undefined, + height?: number | undefined, + codec?: string | undefined, + bitrate?: number | undefined, + frameRate?: number | undefined + ): VideoRenditionLike; + removeRendition(rendition: VideoRenditionLike): void; +} + +export interface VideoTrackListLike extends EventTargetLike> { + readonly length: number; + readonly [index: number]: VideoTrackLike; + [Symbol.iterator](): Iterator; + getTrackById(id: string): VideoTrackLike | null; + readonly selectedIndex: number; +} + +export interface MediaAudioTrackCapability { + readonly audioTracks: AudioTrackListLike; + addAudioTrack(kind: string, label?: string, language?: string): AudioTrackLike; + removeAudioTrack(track: AudioTrackLike): void; +} + +export interface MediaVideoTrackCapability { + readonly videoTracks: VideoTrackListLike; + addVideoTrack(kind: string, label?: string, language?: string): VideoTrackLike; + removeVideoTrack(track: VideoTrackLike): void; +} + +// ---------------------------------------- +// Renditions +// ---------------------------------------- + +interface RenditionEventLike extends EventLike { + readonly rendition: Rendition; +} + +interface RenditionListEvents { + addrendition: RenditionEventLike; + removerendition: RenditionEventLike; + change: EventLike; +} + +export interface AudioRenditionLike { + id: string | undefined; + readonly bitrate: number | undefined; + readonly codec: string | undefined; + selected: boolean; +} + +export interface AudioRenditionListLike extends EventTargetLike> { + readonly length: number; + readonly [index: number]: AudioRenditionLike; + [Symbol.iterator](): Iterator; + getRenditionById(id: string): AudioRenditionLike | null; + selectedIndex: number; +} + +export interface VideoRenditionLike { + id: string | undefined; + readonly width: number | undefined; + readonly height: number | undefined; + readonly bitrate: number | undefined; + readonly frameRate: number | undefined; + readonly codec: string | undefined; + selected: boolean; +} + +export interface VideoRenditionListLike extends EventTargetLike> { + readonly length: number; + readonly [index: number]: VideoRenditionLike; + [Symbol.iterator](): Iterator; + getRenditionById(id: string): VideoRenditionLike | null; + selectedIndex: number; +} + +export interface MediaAudioRenditionCapability { + readonly audioRenditions: AudioRenditionListLike; +} + +export interface MediaVideoRenditionCapability { + readonly videoRenditions: VideoRenditionListLike; +} + // ---------------------------------------- // Fullscreen // ---------------------------------------- diff --git a/packages/core/src/dom/media/dash/index.ts b/packages/core/src/dom/media/dash/index.ts index 610722fa..f1947fcd 100644 --- a/packages/core/src/dom/media/dash/index.ts +++ b/packages/core/src/dom/media/dash/index.ts @@ -1,4 +1,5 @@ import * as dashjs from 'dashjs'; +import { MediaTracksMixin } from '../../../core/media/media-tracks'; import type { MediaEngineHost } from '../../../core/media/types'; import { HTMLVideoElementHost } from '../video-host'; @@ -11,7 +12,7 @@ export const dashMediaDefaultProps: DashMediaProps = { }; export class DashMedia - extends HTMLVideoElementHost + extends MediaTracksMixin(HTMLVideoElementHost) implements MediaEngineHost, DashMediaProps { #engine: dashjs.MediaPlayerClass; diff --git a/packages/core/src/dom/media/hls/hlsjs.ts b/packages/core/src/dom/media/hls/hlsjs.ts index d8584dab..58f2a4bf 100644 --- a/packages/core/src/dom/media/hls/hlsjs.ts +++ b/packages/core/src/dom/media/hls/hlsjs.ts @@ -1,8 +1,17 @@ +import type { MixinReturn } from '@videojs/utils/types'; import Hls, { type HlsConfig } from 'hls.js'; -import type { MediaEngineHost } from '../../../core/media/types'; +import type { MediaError } from '../../../core/media/media-error'; +import { MediaTracksMixin, type WithMediaTracks } from '../../../core/media/media-tracks'; +import type { + MediaEngineHost, + MediaLiveCapability, + MediaSourceCapability, + MediaStreamTypeCapability, +} from '../../../core/media/types'; import { HTMLVideoElementHost } from '../video-host'; import { HlsJsMediaErrorsMixin } from './errors'; import { HlsJsMediaLiveMixin } from './live'; +import { HlsJsMediaMediaTracksMixin } from './media-tracks'; import { HlsJsMediaMetadataTracksMixin } from './metadata-tracks'; import { HlsJsMediaPreloadMixin } from './preload'; import { HlsJsMediaStreamTypeMixin } from './stream-type'; @@ -57,10 +66,23 @@ class HlsJsMediaBase extends HTMLVideoElementHost implements MediaEngineHost { + readonly error: MediaError | null; +} + +const HlsJsMediaComposed = HlsJsMediaPreloadMixin( HlsJsMediaLiveMixin( HlsJsMediaStreamTypeMixin( - HlsJsMediaMetadataTracksMixin(HlsJsMediaTextTracksMixin(HlsJsMediaErrorsMixin(HlsJsMediaBase))) + HlsJsMediaMediaTracksMixin( + HlsJsMediaMetadataTracksMixin( + HlsJsMediaTextTracksMixin(HlsJsMediaErrorsMixin(MediaTracksMixin(HlsJsMediaBase))) + ) + ) ) ) -) {} +) as unknown as MixinReturn, HlsJsMediaCapabilities>; + +export class HlsJsMedia extends HlsJsMediaComposed {} diff --git a/packages/core/src/dom/media/hls/index.ts b/packages/core/src/dom/media/hls/index.ts index af8972c9..7c6d4828 100644 --- a/packages/core/src/dom/media/hls/index.ts +++ b/packages/core/src/dom/media/hls/index.ts @@ -101,6 +101,22 @@ export class HlsMedia extends HTMLVideoElementHost implements HlsMediaProps { return this.#delegate?.error ?? null; } + get videoTracks() { + return this.#delegate instanceof HlsJsMedia ? this.#delegate.videoTracks : undefined; + } + + get audioTracks() { + return this.#delegate instanceof HlsJsMedia ? this.#delegate.audioTracks : undefined; + } + + get videoRenditions() { + return this.#delegate instanceof HlsJsMedia ? this.#delegate.videoRenditions : undefined; + } + + get audioRenditions() { + return this.#delegate instanceof HlsJsMedia ? this.#delegate.audioRenditions : undefined; + } + get src() { return this.#src; } diff --git a/packages/core/src/dom/media/hls/media-tracks.ts b/packages/core/src/dom/media/hls/media-tracks.ts new file mode 100644 index 00000000..296104e8 --- /dev/null +++ b/packages/core/src/dom/media/hls/media-tracks.ts @@ -0,0 +1,177 @@ +import type { Constructor } from '@videojs/utils/types'; +import Hls from 'hls.js'; +import type { + MediaAudioTrackCapability, + MediaVideoRenditionCapability, + MediaVideoTrackCapability, + VideoTrackLike, +} from '../../../core/media/types'; +import type { HlsEngineHost } from './types'; + +type MediaTracksHost = HlsEngineHost & + MediaVideoTrackCapability & + MediaAudioTrackCapability & + MediaVideoRenditionCapability; + +type HlsJsMediaTrackLevel = { + url: string[]; + width?: number | undefined; + height?: number | undefined; + videoCodec?: string | undefined; + bitrate?: number | undefined; +}; + +type HlsJsMediaAudioTrack = { + id: number; + default?: boolean | undefined; + name?: string | undefined; + lang?: string | undefined; +}; + +// hls.js may rebuild its levels array with new object references between +// events, so key levels by their identifying attributes instead of identity. +function getLevelKey(level: HlsJsMediaTrackLevel): string { + return `${level.url[0] ?? ''}|${level.width}x${level.height}|${level.videoCodec}|${level.bitrate}`; +} + +/** + * Mirrors hls.js manifest levels and alternate audio into the media element's + * `videoRenditions` / `audioTracks` lists, and wires user selection back to + * `engine.nextLevel` and `engine.audioTrack`. + * + * Requires the media-tracks mixin (track-list infrastructure) to be applied + * earlier in the chain so the host exposes `addVideoTrack`, `videoRenditions`, + * and friends. + */ +export function HlsJsMediaMediaTracksMixin>(BaseClass: Base) { + class HlsJsMediaMediaTracks extends (BaseClass as Constructor) { + #levelIdMap = new Map(); + #currentVideoTrack: VideoTrackLike | null = null; + + constructor(...args: any[]) { + super(...args); + + const { engine } = this; + if (!engine) return; + + engine.on(Hls.Events.MANIFEST_PARSED, this.#onManifestParsed); + engine.on(Hls.Events.AUDIO_TRACKS_UPDATED, this.#onAudioTracksUpdated); + engine.on(Hls.Events.LEVELS_UPDATED, this.#onLevelsUpdated); + engine.once(Hls.Events.DESTROYING, this.#teardown); + + this.audioTracks.addEventListener('change', this.#switchAudioTrack); + this.videoRenditions.addEventListener('change', this.#switchRendition); + } + + #onManifestParsed = (_event: string, data: { levels: HlsJsMediaTrackLevel[] }) => { + this.#removeAllMediaTracks(); + this.#levelIdMap.clear(); + + const videoTrack = this.addVideoTrack('main'); + this.#currentVideoTrack = videoTrack; + videoTrack.selected = true; + + for (const [id, level] of data.levels.entries()) { + const rendition = videoTrack.addRendition( + level.url[0] ?? '', + level.width, + level.height, + level.videoCodec, + level.bitrate + ); + + this.#levelIdMap.set(getLevelKey(level), `${id}`); + rendition.id = `${id}`; + } + }; + + #onAudioTracksUpdated = (_event: string, data: { audioTracks: HlsJsMediaAudioTrack[] }) => { + this.#removeAudioTracks(); + + for (const hlsAudioTrack of data.audioTracks) { + const kind = hlsAudioTrack.default ? 'main' : 'alternative'; + const audioTrack = this.addAudioTrack(kind, hlsAudioTrack.name, hlsAudioTrack.lang); + audioTrack.id = `${hlsAudioTrack.id}`; + audioTrack.enabled = Boolean(hlsAudioTrack.default); + } + }; + + #switchAudioTrack = () => { + const { engine } = this; + if (!engine) return; + + // `enabled` is not exclusive like video `selected`, so prefer a newly + // enabled track over the one that is already playing. + const enabledTracks = [...this.audioTracks].filter((track) => track.enabled); + const selectedTrack = enabledTracks.find((track) => Number(track.id) !== engine.audioTrack) ?? enabledTracks[0]; + if (!selectedTrack?.id) return; + + const audioTrackId = Number(selectedTrack.id); + const availableIds = engine.audioTracks.map((track) => track.id); + + if (audioTrackId !== engine.audioTrack && availableIds.includes(audioTrackId)) { + engine.audioTrack = audioTrackId; + } + + // Disable the rest so future change events resolve unambiguously. + for (const track of enabledTracks) { + if (track !== selectedTrack) track.enabled = false; + } + }; + + #onLevelsUpdated = (_event: string, data: { levels: HlsJsMediaTrackLevel[] }) => { + if (!this.#currentVideoTrack) return; + + const levelIds = data.levels.map((level) => this.#levelIdMap.get(getLevelKey(level))); + + for (const rendition of this.videoRenditions) { + if (rendition.id && !levelIds.includes(rendition.id)) { + this.#currentVideoTrack.removeRendition(rendition); + } + } + }; + + #switchRendition = () => { + const { engine } = this; + if (!engine) return; + + const level = this.videoRenditions.selectedIndex; + if (level !== engine.nextLevel) engine.nextLevel = level; + }; + + #teardown = () => { + const { engine } = this; + + engine?.off(Hls.Events.MANIFEST_PARSED, this.#onManifestParsed); + engine?.off(Hls.Events.AUDIO_TRACKS_UPDATED, this.#onAudioTracksUpdated); + engine?.off(Hls.Events.LEVELS_UPDATED, this.#onLevelsUpdated); + engine?.off(Hls.Events.DESTROYING, this.#teardown); + + this.audioTracks.removeEventListener('change', this.#switchAudioTrack); + this.videoRenditions.removeEventListener('change', this.#switchRendition); + + this.#removeAllMediaTracks(); + this.#levelIdMap.clear(); + this.#currentVideoTrack = null; + }; + + #removeAllMediaTracks() { + this.#removeVideoTracks(); + this.#removeAudioTracks(); + } + + #removeVideoTracks() { + for (const videoTrack of this.videoTracks) { + this.removeVideoTrack(videoTrack); + } + } + + #removeAudioTracks() { + for (const audioTrack of this.audioTracks) { + this.removeAudioTrack(audioTrack); + } + } + } + + return HlsJsMediaMediaTracks as unknown as Base; +} diff --git a/packages/core/src/dom/media/hls/tests/media-tracks.test.ts b/packages/core/src/dom/media/hls/tests/media-tracks.test.ts new file mode 100644 index 00000000..02b7b01c --- /dev/null +++ b/packages/core/src/dom/media/hls/tests/media-tracks.test.ts @@ -0,0 +1,175 @@ +import Hls from 'hls.js'; +import { describe, expect, it } from 'vitest'; +import { MediaTracksMixin } from '../../../../core/media/media-tracks'; +import { HTMLVideoElementHost } from '../../video-host'; +import { HlsJsMediaMediaTracksMixin } from '../media-tracks'; + +class FakeHost extends HTMLVideoElementHost { + engine: Hls | null; + + constructor(engine: Hls | null = null) { + super(); + this.engine = engine; + } +} + +const HlsJsMediaMediaTracks = HlsJsMediaMediaTracksMixin(MediaTracksMixin(FakeHost)); + +function createEngine(): Hls { + const listeners = new Map void>>(); + return { + audioTracks: [{ id: 0 }, { id: 1 }], + audioTrack: 0, + nextLevel: -1, + on(event: string, fn: (...args: any[]) => void) { + if (!listeners.has(event)) listeners.set(event, new Set()); + listeners.get(event)!.add(fn); + }, + once(event: string, fn: (...args: any[]) => void) { + const wrapped = (...args: any[]) => { + listeners.get(event)?.delete(wrapped); + fn(...args); + }; + if (!listeners.has(event)) listeners.set(event, new Set()); + listeners.get(event)!.add(wrapped); + }, + off(event: string, fn: (...args: any[]) => void) { + listeners.get(event)?.delete(fn); + }, + emit(event: string, ...args: any[]) { + for (const fn of [...(listeners.get(event) ?? [])]) fn(event, ...args); + }, + } as unknown as Hls; +} + +const manifestParsed = (engine: Hls, levels: Array>) => + (engine as any).emit(Hls.Events.MANIFEST_PARSED, { levels }); + +const audioTracksUpdated = (engine: Hls, audioTracks: Array>) => + (engine as any).emit(Hls.Events.AUDIO_TRACKS_UPDATED, { audioTracks }); + +function flush() { + return Promise.resolve(); +} + +describe('HlsJsMediaMediaTracksMixin', () => { + it('mirrors manifest levels into a selected video track with renditions', () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + manifestParsed(engine, [ + { url: ['high.m3u8'], width: 1920, height: 1080, videoCodec: 'avc1', bitrate: 6_000_000 }, + { url: ['low.m3u8'], width: 640, height: 360, videoCodec: 'avc1', bitrate: 800_000 }, + ]); + + expect(host.videoTracks.length).toBe(1); + expect(host.videoTracks[0]?.selected).toBe(true); + expect(host.videoRenditions.length).toBe(2); + expect([...host.videoRenditions].map((rendition) => rendition.id)).toEqual(['0', '1']); + expect([...host.videoRenditions].map((rendition) => rendition.height)).toEqual([1080, 360]); + }); + + it('mirrors alternate audio tracks and enables the default', () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + audioTracksUpdated(engine, [ + { id: 0, default: true, name: 'English', lang: 'en' }, + { id: 1, name: 'Spanish', lang: 'es' }, + ]); + + expect(host.audioTracks.length).toBe(2); + expect(host.audioTracks[0]?.enabled).toBe(true); + expect(host.audioTracks[1]?.enabled).toBe(false); + expect(host.audioTracks[1]?.label).toBe('Spanish'); + }); + + it('forwards a rendition selection to engine.nextLevel', async () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + manifestParsed(engine, [{ url: ['a'] }, { url: ['b'] }, { url: ['c'] }]); + + host.videoRenditions.selectedIndex = 2; + await flush(); + + expect(engine.nextLevel).toBe(2); + }); + + it('forwards an audio track selection to engine.audioTrack', async () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + audioTracksUpdated(engine, [ + { id: 0, default: true, name: 'English', lang: 'en' }, + { id: 1, name: 'Spanish', lang: 'es' }, + ]); + + const [english, spanish] = [...host.audioTracks]; + english!.enabled = false; + spanish!.enabled = true; + await flush(); + + expect(engine.audioTrack).toBe(1); + }); + + it('prunes renditions dropped from a LEVELS_UPDATED event', () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + const levels = [{ url: ['a'] }, { url: ['b'] }, { url: ['c'] }]; + manifestParsed(engine, levels); + expect(host.videoRenditions.length).toBe(3); + + (engine as any).emit(Hls.Events.LEVELS_UPDATED, { levels: [levels[0], levels[2]] }); + + expect([...host.videoRenditions].map((rendition) => rendition.id)).toEqual(['0', '2']); + }); + + it('prunes renditions when LEVELS_UPDATED carries new level object references', () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + const levels = [{ url: ['a'] }, { url: ['b'] }, { url: ['c'] }]; + manifestParsed(engine, levels); + expect(host.videoRenditions.length).toBe(3); + + (engine as any).emit(Hls.Events.LEVELS_UPDATED, { levels: [{ ...levels[0] }, { ...levels[2] }] }); + + expect([...host.videoRenditions].map((rendition) => rendition.id)).toEqual(['0', '2']); + }); + + it('switches to the newly enabled audio track when multiple are enabled', async () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + audioTracksUpdated(engine, [ + { id: 0, default: true, name: 'English', lang: 'en' }, + { id: 1, name: 'Spanish', lang: 'es' }, + ]); + + const [english, spanish] = [...host.audioTracks]; + // Enable without disabling the current track first. + spanish!.enabled = true; + await flush(); + + expect(engine.audioTrack).toBe(1); + expect(english!.enabled).toBe(false); + expect(spanish!.enabled).toBe(true); + }); + + it('clears all media tracks on DESTROYING', () => { + const engine = createEngine(); + const host = new HlsJsMediaMediaTracks(engine); + + manifestParsed(engine, [{ url: ['a'] }]); + audioTracksUpdated(engine, [{ id: 0, default: true }]); + expect(host.videoTracks.length).toBe(1); + expect(host.audioTracks.length).toBe(1); + + (engine as any).emit(Hls.Events.DESTROYING); + + expect(host.videoTracks.length).toBe(0); + expect(host.audioTracks.length).toBe(0); + }); +}); diff --git a/packages/core/src/dom/store/features/remote-playback.ts b/packages/core/src/dom/store/features/remote-playback.ts index 045e3f46..ff1367d6 100644 --- a/packages/core/src/dom/store/features/remote-playback.ts +++ b/packages/core/src/dom/store/features/remote-playback.ts @@ -74,7 +74,7 @@ export const remotePlaybackFeature = definePlayerFeature({ }); signal.addEventListener('abort', () => { - media.remote?.cancelWatchAvailability().catch(() => {}); + media.remote?.cancelWatchAvailability?.().catch(() => {}); }); }, }); diff --git a/packages/core/src/dom/store/features/tests/remote-playback.test.ts b/packages/core/src/dom/store/features/tests/remote-playback.test.ts new file mode 100644 index 00000000..2dd5acb9 --- /dev/null +++ b/packages/core/src/dom/store/features/tests/remote-playback.test.ts @@ -0,0 +1,64 @@ +import type { AttachContext } from '@videojs/store'; +import { describe, expect, it, vi } from 'vitest'; +import type { Media, PlayerTarget } from '../../../media/types'; +import { remotePlaybackFeature } from '../remote-playback'; + +type RemotePlaybackState = ReturnType; + +function createRemote(overrides: Partial = {}) { + const target = new EventTarget(); + return Object.assign(target, { + state: 'disconnected', + watchAvailability: vi.fn().mockResolvedValue(1), + cancelWatchAvailability: vi.fn().mockResolvedValue(undefined), + ...overrides, + }); +} + +interface RemotePlaybackLike { + state: string; + watchAvailability: (callback: (available: boolean) => void) => Promise; + cancelWatchAvailability?: (id?: number) => Promise; +} + +function attach(media: Media) { + const controller = new AbortController(); + const set = vi.fn(); + + remotePlaybackFeature.attach?.({ + target: { media, container: null } as PlayerTarget, + signal: controller.signal, + set, + get: () => ({}) as RemotePlaybackState, + store: { state: {}, subscribe: () => () => {} }, + reportError: () => {}, + } as AttachContext); + + return { controller, set }; +} + +describe('remotePlaybackFeature', () => { + it('cancels availability watching on abort (W3C path)', () => { + const remote = createRemote(); + const media = { remote } as unknown as Media; + + const { controller } = attach(media); + controller.abort(); + + expect(remote.cancelWatchAvailability).toHaveBeenCalledOnce(); + }); + + it('does not throw on abort when cancelWatchAvailability becomes unavailable', () => { + // Simulate a custom element whose `remote` resolves to a partial object at + // detach time — the captured reference must guard the method call. + const remote = createRemote(); + const media = { remote } as unknown as Media; + + const { controller } = attach(media); + + // The W3C RemotePlayback object loses its method (e.g. inner video torn down). + (remote as { cancelWatchAvailability?: unknown }).cancelWatchAvailability = undefined; + + expect(() => controller.abort()).not.toThrow(); + }); +}); diff --git a/packages/html/src/store/media-attach-mixin.ts b/packages/html/src/store/media-attach-mixin.ts index eb3c9c87..5b8e123a 100644 --- a/packages/html/src/store/media-attach-mixin.ts +++ b/packages/html/src/store/media-attach-mixin.ts @@ -45,11 +45,15 @@ export function createMediaAttachMixin(context: MediaContext): MediaAttachMixin } override disconnectedCallback() { - super.disconnectedCallback?.(); + // Detach the store while the media chain is still live so features + // (e.g. remote-playback) can clean up against the real underlying + // element. Destroying the media host first would null the layer + // chain's target before listeners get a chance to unwind. this.#setMedia?.(null); this.#unsubscribe?.(); this.#unsubscribe = null; this.#setMedia = null; + super.disconnectedCallback?.(); } }