feat(spf): multi cdn failover (#1671)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Christian Pillsbury
2026-06-17 08:20:43 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 33873bc4ae
commit b89f1e944c
29 changed files with 1508 additions and 248 deletions
+6 -2
View File
@@ -10,6 +10,7 @@ src/
media/ CML-like media building blocks — types, parsers, ABR, buffer logic, MSE/VTT primitives
network/ HTTP fetch utilities, chunked-stream iterables
playback/ Playback domain — composes core+media+network into engines
primitives/ Signal-aware playback building blocks composed by behaviors (no behavior lifecycle)
behaviors/ Compositional behaviors that drive playback (use signals/effects)
actors/ Actor factories used inside behaviors
engines/ Assembled playback engines (currently just hls/)
@@ -31,8 +32,10 @@ src/
| `core/` → DOM | ❌ | `core/tsconfig.json` `lib` excludes `DOM` |
| `media/` (non-dom) → DOM | ❌ | `media/tsconfig.json` `lib` excludes `DOM` |
| `network/` → DOM | ❌ | `network/tsconfig.json` `lib` excludes `DOM` |
| `playback/primitives/` → DOM | ❌ | tsconfig `lib` excludes `DOM` |
| `playback/behaviors/` (non-dom) → DOM | ❌ | tsconfig `lib` excludes `DOM` |
| `playback/actors/` (non-dom) → DOM | ❌ | tsconfig `lib` excludes `DOM` |
| `playback/behaviors/``playback/primitives/` | ✅ | references |
| `playback/``core/`, `network/`, `media/` | ✅ | references in playback tsconfigs |
| `playback/engines/hls/``playback/behaviors/`, `playback/actors/` | ✅ | references |
| `playback/engines/hls/``core/`, `media/` | ✅ — engines compose primitives directly | references |
@@ -42,12 +45,13 @@ The substance: `core/`, `media/`, `network/` are framework-agnostic foundations.
## Where to put new code
- **Pure media/streaming logic** (parsers, types, selection algorithms, MSE/VTT helpers without signals): `media/` or `media/dom/`. Must not import from `core/`.
- **Signal-aware playback primitive** (a composable building block that reads/writes `core` signals but has no behavior lifecycle — no `effect`/`computed`/`subscribe`, not invoked by composition): `playback/primitives/`. These are composed *by* behaviors/actors/engines (e.g. `failoverFetch`, a fetch decorator that reads selected-track signals at fetch time; `track-types`, the per-type config bundles). They live at the `playback/` layer rather than `media/`/`network/` precisely because they touch `core/` — that's the line below.
- **Compositional behavior driving state** (uses `effect`/`computed`/`update` against owners or state signals): `playback/behaviors/` or `playback/behaviors/dom/`.
- **Actor factories** (long-lived stateful units that receive messages): `playback/actors/` or `playback/actors/dom/`.
- **Engine compositions** (wiring behaviors+actors+config into a `createComposition` call): `playback/engines/<name>/`.
- **Generic, framework-agnostic utilities** that aren't media-specific: prefer `@videojs/utils` over creating new homes inside spf.
If a module looks like a primitive but reaches into `core/`, that's a smell — consider whether the signal binding can move to the call site (see `onMediaSourceReadyStateChange` for a callback-shaped primitive that lets the consumer create the signal).
If a module looks like a `media/`/`network/` primitive but reaches into `core/`, that's a smell — first consider whether the signal binding can move to the call site (see `onMediaSourceReadyStateChange` for a callback-shaped primitive that lets the consumer create the signal), keeping the primitive itself `core`-free. Only when a building block genuinely needs *live* signal access — e.g. `failoverFetch` is constructed once but its returned fetch reads `presentation`/selected-track signals lazily on every call — does it belong at the `playback/` layer, in `playback/primitives/`.
Conversely: if a function inside `playback/behaviors/` (or `playback/actors/`) has no `core/` dependency — no signals, effects, or reactors — it probably belongs in a layer below (`media/`, `network/`, or `@videojs/utils`). Same layering principle, opposite direction. When reviewing a behavior or actor file, scan its top-level helpers; any pure data-manipulation / lookup / format-handling code with no reactive concerns is a candidate to extract.
@@ -71,7 +75,7 @@ Internal paths are not part of the public API. Don't import from `@videojs/spf/p
## Vitest projects
`packages/spf/vitest.config.ts` shards tests by area:
- `core`, `media`, `network`, `behaviors` — Node, no browser
- `core`, `media`, `network`, `behaviors` — Node, no browser (the `behaviors` project also covers `playback/actors/` and `playback/primitives/`)
- `dom` — Chromium via Playwright, covers all `**/dom/**/*.test.ts` across subtrees
- `playback-engines` — Chromium, covers engines/
- `types` — type-only tests via tsgo
+28 -10
View File
@@ -1,13 +1,19 @@
import type { MaybeResolvedPresentation, TrackType } from '../types';
/**
* Identify the CDN a URL is served from, used to group redundant-stream
* variants that point at the same content on different hosts. Defaults to the
* URL's origin (scheme + host + port); falls back to the raw string when the
* URL can't be parsed, so the return value is always a stable grouping key.
*
* Sub-feature 1 (sticky CDN pick) uses origin-based identity; a more advanced
* or consumer-configurable derivation can replace this default later.
* Derive a stable grouping key for the CDN a URL is served from. Synchronous and
* pure (deliberately not a `resolve*` — no fetch). Consumers override the
* default via the engine's `getCdnId` config (e.g. to key on Mux's `cdn=` query
* param instead of the host); every CDN-identity site reads that same function
* so keys stay comparable across `cdnPriority`, `failedCdns`, and the
* track-switching constraint + scope.
*/
export type GetCdnId = (url: string) => string;
/**
* Default {@link GetCdnId}: the URL's origin (scheme + host + port); falls back
* to the raw string when the URL can't be parsed, so the return value is always
* a stable grouping key.
*/
export function getCdnId(url: string): string {
try {
@@ -34,9 +40,11 @@ const CDN_TYPE_PRIORITY: Record<TrackType, number> = { video: 0, audio: 1, text:
*
* Redundant-stream sources list the same content on multiple hosts (e.g. Mux's
* `?redundant_streams=true`), so each host contributes its own candidate tracks;
* this collapses them to the set of CDNs across every track type.
* this collapses them to the set of CDNs across every track type. The CDN-id
* derivation defaults to {@link getCdnId}; pass a consumer-configured `getId` to
* key on something other than origin.
*/
export function getOrderedCdnIds(presentation: MaybeResolvedPresentation): string[] {
export function getOrderedCdnIds(presentation: MaybeResolvedPresentation, getId: GetCdnId = getCdnId): string[] {
const seen = new Set<string>();
const ids: string[] = [];
// Stable sort keeps manifest order among same-type selection sets.
@@ -46,7 +54,7 @@ export function getOrderedCdnIds(presentation: MaybeResolvedPresentation): strin
for (const selectionSet of selectionSets) {
for (const switchingSet of selectionSet.switchingSets) {
for (const track of switchingSet.tracks) {
const id = getCdnId(track.url);
const id = getId(track.url);
if (seen.has(id)) continue;
seen.add(id);
ids.push(id);
@@ -55,3 +63,13 @@ export function getOrderedCdnIds(presentation: MaybeResolvedPresentation): strin
}
return ids;
}
/**
* Add a CDN id to a failed-CDN list, preserving order and ignoring duplicates.
* Idempotent: re-adding an already-present id returns the same array reference
* (so a no-op trip doesn't churn the `failedCdns` signal). The failover trip in
* `resolve-track` and the segment loaders feed this into `failedCdns` via `update`.
*/
export function addFailedCdn(failed: string[] | undefined, cdn: string): string[] {
return failed?.includes(cdn) ? failed : [...(failed ?? []), cdn];
}
+20 -1
View File
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { MaybeResolvedPresentation } from '../../types';
import { getCdnId, getOrderedCdnIds } from '../cdn';
import { addFailedCdn, getCdnId, getOrderedCdnIds } from '../cdn';
const presentationWith = (urlsByType: {
video?: string[];
@@ -119,3 +119,22 @@ describe('getOrderedCdnIds', () => {
expect(getOrderedCdnIds(presentation)).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']);
});
});
describe('addFailedCdn', () => {
it('appends to an undefined or empty list', () => {
expect(addFailedCdn(undefined, 'https://cdn-a.example.com')).toEqual(['https://cdn-a.example.com']);
expect(addFailedCdn([], 'https://cdn-a.example.com')).toEqual(['https://cdn-a.example.com']);
});
it('appends in order', () => {
expect(addFailedCdn(['https://cdn-a.example.com'], 'https://cdn-b.example.com')).toEqual([
'https://cdn-a.example.com',
'https://cdn-b.example.com',
]);
});
it('is idempotent — re-adding a present CDN returns the same array reference', () => {
const failed = ['https://cdn-a.example.com'];
expect(addFailedCdn(failed, 'https://cdn-a.example.com')).toBe(failed);
});
});
+17
View File
@@ -124,6 +124,23 @@ export function getResponseText(response: ResponseLike): Promise<string> {
return response.text();
}
/**
* Fetch a resource and resolve its text body — the text analog of
* {@link FetchBytes}. A non-OK status rejects, so HTTP failures surface as
* rejections that callers (and decorators like the failover tracker) handle
* uniformly with network errors.
*/
export type FetchText = (addressable: Resource, options?: RequestInit) => Promise<string>;
/** Default {@link FetchText}: fetch the resource, reject on non-OK, return text. */
export const fetchResolvableText: FetchText = async (addressable, options) => {
const response = await fetchResolvable(addressable, options);
if (!response.ok) {
throw new Error(`fetchResolvableText: ${response.status} ${response.statusText} for ${addressable.url}`);
}
return getResponseText(response);
};
/**
* Two-stage fetch helper: eagerly starts the HTTP request (TTFB is awaited),
* then returns a lazy iterable over the response body. Separating connection
@@ -29,9 +29,9 @@ import { defineBehavior } from '../../core/composition/create-composition';
import { createMachineReactor } from '../../core/reactors/create-machine-reactor';
import { computed, peek, type ReadonlySignal, type Signal } from '../../core/signals/primitives';
import { isResolvedPresentation, type MaybeResolvedPresentation } from '../../media/types';
import { getOrderedCdnIds } from '../../media/utils/cdn';
import { getCdnId as defaultGetCdnId, type GetCdnId, getOrderedCdnIds } from '../../media/utils/cdn';
export interface ResolveCdnPriorityState {
export interface DeriveCdnPriorityState {
presentation?: MaybeResolvedPresentation;
cdnPriority?: string[];
}
@@ -44,19 +44,22 @@ const samePriority = (a: string[] | undefined, b: string[]): boolean =>
* on src unload.
*
* @example
* const reactor = resolveCdnPriority.setup({ state });
* const reactor = deriveCdnPriority.setup({ state });
*/
export const resolveCdnPriority = defineBehavior({
export const deriveCdnPriority = defineBehavior({
stateKeys: ['presentation', 'cdnPriority'],
contextKeys: [],
setup: ({
state,
config = {},
}: {
state: {
presentation: ReadonlySignal<ResolveCdnPriorityState['presentation']>;
cdnPriority: Signal<ResolveCdnPriorityState['cdnPriority']>;
presentation: ReadonlySignal<DeriveCdnPriorityState['presentation']>;
cdnPriority: Signal<DeriveCdnPriorityState['cdnPriority']>;
};
config?: { getCdnId?: GetCdnId };
}) => {
const getCdnId = config.getCdnId ?? defaultGetCdnId;
const derivedStateSignal = computed(() =>
isResolvedPresentation(state.presentation.get())
? ('presentation-resolved' as const)
@@ -76,7 +79,7 @@ export const resolveCdnPriority = defineBehavior({
() => {
const presentation = state.presentation.get();
if (!isResolvedPresentation(presentation)) return;
const next = getOrderedCdnIds(presentation);
const next = getOrderedCdnIds(presentation, getCdnId);
// Skip the write when the CDN set is unchanged — a live reload swaps
// in a new presentation object with the same hosts, and re-setting a
// fresh array would re-fire the scope for an identical result.
@@ -56,7 +56,7 @@ import type { MaybeResolvedPresentation, Segment } from '../../../media/types';
import { findResolvedAudioTrack, findResolvedTextTrack, findResolvedVideoTrack } from '../../../media/utils/tracks';
import type { BufferState, SegmentLoaderActor, SourceBufferState } from '../../actors/dom/segment-loader';
import type { TextTrackSegmentLoaderActor } from '../../actors/text-track-segment-loader';
import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../track-types';
import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../../primitives/track-types';
// Re-export buffer state types for consumers that import them from this module.
export type { BufferState, SourceBufferState };
@@ -65,6 +65,7 @@ import { createMachineReactor } from '../../../core/reactors/create-machine-reac
import { computed, type ReadonlySignal, type Signal } from '../../../core/signals/primitives';
import { buildMimeCodec, createSourceBuffer } from '../../../media/dom/mse/mediasource-setup';
import type { MaybeResolvedPresentation, PartiallyResolvedTrack } from '../../../media/types';
import type { GetCdnId } from '../../../media/utils/cdn';
import { getSelectedTrack, type TrackSelectionState } from '../../../media/utils/track-selection';
import { hasCodecs } from '../../../media/utils/tracks';
import type { BandwidthState } from '../../../network/bandwidth-estimator';
@@ -75,7 +76,8 @@ import {
type SegmentLoaderActorConfig,
} from '../../actors/dom/segment-loader';
import { createSourceBufferActor, type SourceBufferActor } from '../../actors/dom/source-buffer';
import { AUDIO_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../track-types';
import { failoverFetch } from '../../primitives/failover-fetch';
import { AUDIO_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../../primitives/track-types';
/**
* Media track type for MSE buffer setup.
@@ -221,7 +223,7 @@ export const setupVideoBufferActors = defineBehavior({
bandwidthState: Signal<BufferActorsState['bandwidthState']>;
};
context: BufferActorsContextMap<'videoBufferActor', 'videoSegmentLoaderActor'>;
config?: object;
config?: SegmentLoaderActorConfig & { getCdnId?: GetCdnId };
}) => {
// Bandwidth-sampling fetch. The factory accumulates EWMA state
// internally; the callback bridges samples to engine state for ABR.
@@ -238,10 +240,13 @@ export const setupVideoBufferActors = defineBehavior({
},
(next) => state.bandwidthState.set(next)
);
// Engine `config` layers over the per-type defaults; `failoverFetch` reads
// its `selectedKey` + `getCdnId` from the merged result.
const typeConfig = { ...VIDEO_TYPE_CONFIG, ...config };
return setupBufferActors({
state,
context,
config: { ...VIDEO_TYPE_CONFIG, fetch: trackedFetch, ...config },
config: { ...typeConfig, fetch: failoverFetch(trackedFetch, state, typeConfig) },
});
},
});
@@ -274,11 +279,14 @@ export const setupAudioBufferActors = defineBehavior({
}: {
state: BufferActorsStateMap<'selectedAudioTrackId'>;
context: BufferActorsContextMap<'audioBufferActor', 'audioSegmentLoaderActor'>;
config?: object;
}) =>
setupBufferActors({
config?: SegmentLoaderActorConfig & { getCdnId?: GetCdnId };
}) => {
// Key order mirrors setupVideoBufferActors.
const typeConfig = { ...AUDIO_TYPE_CONFIG, ...config };
return setupBufferActors({
state,
context,
config: { ...AUDIO_TYPE_CONFIG, fetch: fetchStream, ...config },
}),
config: { ...typeConfig, fetch: failoverFetch(fetchStream, state, typeConfig) },
});
},
});
@@ -12,6 +12,7 @@
{ "path": "../../../network" },
{ "path": "../../../media" },
{ "path": "../../../media/dom" },
{ "path": "../../primitives" },
{ "path": "../" },
{ "path": "../../actors" },
{ "path": "../../actors/dom" }
@@ -5,9 +5,11 @@ import { ConcurrentRunner, Task } from '../../core/tasks/task';
import { parseMediaPlaylist } from '../../media/hls/parse-media-playlist';
import type { MaybeResolvedPresentation, PartiallyResolvedTrack, ResolvedTrack } from '../../media/types';
import { isResolvedPresentation, isResolvedTrack } from '../../media/types';
import type { GetCdnId } from '../../media/utils/cdn';
import { findTrack, updateTrackInPresentation } from '../../media/utils/tracks';
import { fetchResolvable, getResponseText } from '../../network/fetch';
import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from './track-types';
import { fetchResolvableText as defaultFetchResolvableText, type FetchText } from '../../network/fetch';
import { failoverFetch } from '../primitives/failover-fetch';
import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../primitives/track-types';
// ============================================================================
// Specialization helper
@@ -29,6 +31,7 @@ export interface ResolveTrackState {
selectedVideoTrackId?: string;
selectedAudioTrackId?: string;
selectedTextTrackId?: string;
failedCdns?: string[];
}
type SelectedTrackKey = 'selectedVideoTrackId' | 'selectedAudioTrackId' | 'selectedTextTrackId';
@@ -43,11 +46,22 @@ interface TrackResolutionConfig<K extends SelectedTrackKey> {
presentation: MaybeResolvedPresentation,
trackId: string
) => PartiallyResolvedTrack | ResolvedTrack | undefined;
/** Fetch a track's media-playlist text — already failover-decorated by the behavior. */
fetchResolvableText?: FetchText;
}
/**
* Engine-config slice each `resolve*` behavior reads to build its failover-
* decorated playlist fetch.
*/
interface ResolveTrackConfig {
/** CDN-id derivation for the failover trip; defaults to origin-based `getCdnId`. */
getCdnId?: GetCdnId;
}
function setupTrackResolution<K extends SelectedTrackKey>({
state,
config: { selectedKey, findTrackToResolve },
config: { selectedKey, findTrackToResolve, fetchResolvableText = defaultFetchResolvableText },
}: {
state: ResolveTrackStateMap<K>;
config: TrackResolutionConfig<K>;
@@ -104,8 +118,11 @@ function setupTrackResolution<K extends SelectedTrackKey>({
// likely eventually passed down via config or a new "definitions" argument (CJP).
new Task(
async (signal) => {
const response = await fetchResolvable(track, { signal });
const text = await getResponseText(response);
// `fetchResolvableText` is the behavior's failover-decorated
// fetch: it trips the CDN on a failed fetch (network error or
// non-OK status). A parse failure is a content issue, not a
// CDN-availability one, so it doesn't trip.
const text = await fetchResolvableText(track, { signal });
const mediaTrack = parseMediaPlaylist(text, track);
// Updater handles undefined inputs by returning current
@@ -163,11 +180,24 @@ const TEXT_TRACK_RESOLUTION_CONFIG = {
export const resolveVideoTrack = defineBehavior({
stateKeys: ['presentation', 'selectedVideoTrackId'],
contextKeys: [],
setup: ({ state, config = {} }: { state: ResolveTrackStateMap<'selectedVideoTrackId'>; config?: object }) =>
setupTrackResolution({
setup: ({
state,
config = {},
}: {
state: ResolveTrackStateMap<'selectedVideoTrackId'>;
config?: ResolveTrackConfig;
}) => {
// Engine `config` layers over the per-type defaults (mirrors the other
// per-type variants, see track-types.ts); `failoverFetch` reads its
// `selectedKey` + `getCdnId` from the merged result. `fetchResolvableText`
// is then placed AFTER the spread so the failover-decorated fetch wins —
// unlike segments, playlists expose no overridable per-type fetch.
const trackConfig = { ...VIDEO_TRACK_RESOLUTION_CONFIG, ...config };
return setupTrackResolution({
state,
config: { ...VIDEO_TRACK_RESOLUTION_CONFIG, ...config },
}),
config: { ...trackConfig, fetchResolvableText: failoverFetch(defaultFetchResolvableText, state, trackConfig) },
});
},
});
/**
@@ -177,11 +207,20 @@ export const resolveVideoTrack = defineBehavior({
export const resolveAudioTrack = defineBehavior({
stateKeys: ['presentation', 'selectedAudioTrackId'],
contextKeys: [],
setup: ({ state, config = {} }: { state: ResolveTrackStateMap<'selectedAudioTrackId'>; config?: object }) =>
setupTrackResolution({
setup: ({
state,
config = {},
}: {
state: ResolveTrackStateMap<'selectedAudioTrackId'>;
config?: ResolveTrackConfig;
}) => {
// Key order is load-bearing — see resolveVideoTrack.
const trackConfig = { ...AUDIO_TRACK_RESOLUTION_CONFIG, ...config };
return setupTrackResolution({
state,
config: { ...AUDIO_TRACK_RESOLUTION_CONFIG, ...config },
}),
config: { ...trackConfig, fetchResolvableText: failoverFetch(defaultFetchResolvableText, state, trackConfig) },
});
},
});
/**
@@ -191,9 +230,18 @@ export const resolveAudioTrack = defineBehavior({
export const resolveTextTrack = defineBehavior({
stateKeys: ['presentation', 'selectedTextTrackId'],
contextKeys: [],
setup: ({ state, config = {} }: { state: ResolveTrackStateMap<'selectedTextTrackId'>; config?: object }) =>
setupTrackResolution({
setup: ({
state,
config = {},
}: {
state: ResolveTrackStateMap<'selectedTextTrackId'>;
config?: ResolveTrackConfig;
}) => {
// Key order is load-bearing — see resolveVideoTrack.
const trackConfig = { ...TEXT_TRACK_RESOLUTION_CONFIG, ...config };
return setupTrackResolution({
state,
config: { ...TEXT_TRACK_RESOLUTION_CONFIG, ...config },
}),
config: { ...trackConfig, fetchResolvableText: failoverFetch(defaultFetchResolvableText, state, trackConfig) },
});
},
});
@@ -44,7 +44,7 @@ import {
type VideoSelectionConfig,
} from '../../media/primitives/select-tracks';
import { isResolvedPresentation } from '../../media/types';
import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from './track-types';
import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../primitives/track-types';
// ============================================================================
// Specialization helper
@@ -0,0 +1,113 @@
/**
* **CDN failover cooldown.** The expiry half of multi-CDN failover. Fetch sites
* own the *trip*: on a failed fetch they add the failing CDN (origin) to the
* `failedCdns` state signal directly. This behavior owns the *expiry*: while a
* presentation is resolved, it watches `failedCdns` and, for each CDN that
* appears, schedules a timer to remove it once its cooldown lapses.
* `track-switching`'s `excludeFailedCdns` constraint prunes a failed CDN's
* tracks and the active-CDN scope falls to the next one and back, once the
* cooldown removes it here.
*
* Lifecycle is per-source: timers + `failedCdns` are cleared on exit (a new
* source starts with a clean slate). Policy (cooldown) is engine config. This is
* the minimal `network-resilience` slice a single failure trips a CDN, since
* transient blips are the retry layer's job (it sits below the fetch sites, so
* anything that reaches `failedCdns` is already terminal).
*/
import { defineBehavior } from '../../core/composition/create-composition';
import { createMachineReactor } from '../../core/reactors/create-machine-reactor';
import { computed, type ReadonlySignal, type Signal, update } from '../../core/signals/primitives';
import { isResolvedPresentation, type MaybeResolvedPresentation } from '../../media/types';
/**
* Failover policy: how long a CDN stays excluded after a failed fetch trips it.
* Supplied via engine config.
*/
export interface FailoverMonitorConfig {
/** How long a tripped CDN stays excluded, in milliseconds. */
cooldownMs: number;
}
export const DEFAULT_FAILOVER_MONITOR_CONFIG: FailoverMonitorConfig = {
// 5 minutes — a CDN outage is an infrastructure problem that outlasts a
// transient blip, so re-probing it sooner mostly re-trips. Matches the
// prevailing prior-art default (ExoPlayer's location exclusion, hls.js's
// content-steering penalty box).
cooldownMs: 300_000,
};
export interface SetupFailoverMonitorState {
presentation?: MaybeResolvedPresentation;
failedCdns?: string[];
}
export interface SetupFailoverMonitorConfig {
/** Failover policy (cooldown); defaults to `DEFAULT_FAILOVER_MONITOR_CONFIG`. */
failover?: Partial<FailoverMonitorConfig>;
}
/**
* Expire failed CDNs from `failedCdns` once their cooldown lapses, for the
* resolved source.
*
* @example
* const reactor = setupFailoverMonitor.setup({ state });
*/
export const setupFailoverMonitor = defineBehavior({
stateKeys: ['presentation', 'failedCdns'],
contextKeys: [],
setup: ({
state,
config = {},
}: {
state: {
presentation: ReadonlySignal<SetupFailoverMonitorState['presentation']>;
failedCdns: Signal<SetupFailoverMonitorState['failedCdns']>;
};
config?: SetupFailoverMonitorConfig;
}) => {
const cooldownMs = config.failover?.cooldownMs ?? DEFAULT_FAILOVER_MONITOR_CONFIG.cooldownMs;
// CDN id → its pending cooldown-removal timer. Shared by the `effects`
// scheduler (adds a timer per newly-failed CDN) and the exit cleanup
// (clears them). Per-source: emptied on exit, so it re-enters clean.
const timers = new Map<string, ReturnType<typeof setTimeout>>();
const derivedStateSignal = computed(() =>
isResolvedPresentation(state.presentation.get())
? ('presentation-resolved' as const)
: ('presentation-unresolved' as const)
);
return createMachineReactor({
initial: 'presentation-unresolved',
monitor: () => derivedStateSignal.get(),
states: {
'presentation-unresolved': {},
'presentation-resolved': {
// Cleanup-binds-to-setup: on exit (src unload + destroy) clear the
// pending timers and reset `failedCdns` for the next source.
entry: () => () => {
timers.forEach((timer) => clearTimeout(timer));
timers.clear();
state.failedCdns.set(undefined);
},
effects: [
() => {
const failed = state.failedCdns.get() ?? [];
failed.forEach((cdn) => {
// Idempotent: a CDN already counting down keeps its original
// deadline (re-failing it mid-cooldown doesn't extend it).
if (timers.has(cdn)) return;
const timer = setTimeout(() => {
timers.delete(cdn);
update(state.failedCdns, (current) => current?.filter((c) => c !== cdn));
}, cooldownMs);
timers.set(cdn, timer);
});
},
],
},
},
});
},
});
@@ -2,9 +2,9 @@ import { describe, expect, it } from 'vitest';
import type { StateSignals } from '../../../core/composition/create-composition';
import { signal } from '../../../core/signals/primitives';
import type { MaybeResolvedPresentation, PartiallyResolvedVideoTrack, Presentation } from '../../../media/types';
import { type ResolveCdnPriorityState, resolveCdnPriority } from '../resolve-cdn-priority';
import { type DeriveCdnPriorityState, deriveCdnPriority } from '../derive-cdn-priority';
function makeState(initial: Partial<ResolveCdnPriorityState> = {}): StateSignals<ResolveCdnPriorityState> {
function makeState(initial: Partial<DeriveCdnPriorityState> = {}): StateSignals<DeriveCdnPriorityState> {
return {
presentation: signal<MaybeResolvedPresentation | undefined>(initial.presentation),
cdnPriority: signal<string[] | undefined>(initial.cdnPriority),
@@ -50,10 +50,10 @@ const redundant = (id = 'pres-1'): Presentation =>
const flush = () => Promise.resolve().then(() => Promise.resolve());
describe('resolveCdnPriority', () => {
describe('deriveCdnPriority', () => {
it('does nothing without a presentation', async () => {
const state = makeState();
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
expect(state.cdnPriority.get()).toBeUndefined();
reactor.destroy();
@@ -61,7 +61,7 @@ describe('resolveCdnPriority', () => {
it('publishes the manifest-ordered CDN list on src load', async () => {
const state = makeState({ presentation: redundant() });
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']);
reactor.destroy();
@@ -69,7 +69,7 @@ describe('resolveCdnPriority', () => {
it('publishes a single-entry list for a non-redundant source', async () => {
const state = makeState({ presentation: presentationWith(['https://cdn-a.example.com/720p.m3u8']) });
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com']);
reactor.destroy();
@@ -77,7 +77,7 @@ describe('resolveCdnPriority', () => {
it('does not re-set the list when a resolved swap keeps the same CDNs', async () => {
const state = makeState({ presentation: redundant() });
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
const first = state.cdnPriority.get();
@@ -92,7 +92,7 @@ describe('resolveCdnPriority', () => {
it('updates the list when a resolved swap changes the CDN order', async () => {
const state = makeState({ presentation: redundant() });
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']);
@@ -107,7 +107,7 @@ describe('resolveCdnPriority', () => {
it('clears cdnPriority on src unload', async () => {
const state = makeState({ presentation: redundant() });
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
expect(state.cdnPriority.get()).toBeDefined();
@@ -120,7 +120,7 @@ describe('resolveCdnPriority', () => {
it('clears cdnPriority on destroy', async () => {
const state = makeState({ presentation: redundant() });
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
expect(state.cdnPriority.get()).toBeDefined();
@@ -130,7 +130,7 @@ describe('resolveCdnPriority', () => {
it('re-publishes after a src reset (undefined → new resolved)', async () => {
const state = makeState({ presentation: redundant() });
const reactor = resolveCdnPriority.setup({ state });
const reactor = deriveCdnPriority.setup({ state });
await flush();
expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']);
@@ -21,6 +21,7 @@ function makeState(initial: ResolveTrackState = {}): StateSignals<ResolveTrackSt
selectedVideoTrackId: signal<string | undefined>(initial.selectedVideoTrackId),
selectedAudioTrackId: signal<string | undefined>(initial.selectedAudioTrackId),
selectedTextTrackId: signal<string | undefined>(initial.selectedTextTrackId),
failedCdns: signal<string[] | undefined>(initial.failedCdns),
};
}
@@ -0,0 +1,83 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { signal } from '../../../core/signals/primitives';
import type { MaybeResolvedPresentation, Presentation } from '../../../media/types';
import { DEFAULT_FAILOVER_MONITOR_CONFIG, setupFailoverMonitor } from '../setup-failover-monitor';
const resolved = (): Presentation =>
({ id: 'pres-1', url: 'https://cdn-a.example.com/master.m3u8', startTime: 0, selectionSets: [] }) as Presentation;
const makeState = (presentation?: MaybeResolvedPresentation) => ({
presentation: signal<MaybeResolvedPresentation | undefined>(presentation),
failedCdns: signal<string[] | undefined>(undefined),
});
const flush = () => Promise.resolve().then(() => Promise.resolve());
const A = 'https://cdn-a.example.com';
const B = 'https://cdn-b.example.com';
describe('setupFailoverMonitor', () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(0);
});
afterEach(() => {
vi.useRealTimers();
});
it('removes a failed CDN once its cooldown lapses', async () => {
const state = makeState(resolved());
const reactor = setupFailoverMonitor.setup({ state, config: { failover: { cooldownMs: 1000 } } });
await flush();
state.failedCdns.set([A]); // a fetch site tripped cdn-a
await flush();
expect(state.failedCdns.get()).toEqual([A]);
await vi.advanceTimersByTimeAsync(1000);
expect(state.failedCdns.get()).toEqual([]);
reactor.destroy();
});
it('expires each CDN on its own cooldown', async () => {
const state = makeState(resolved());
const reactor = setupFailoverMonitor.setup({ state, config: { failover: { cooldownMs: 1000 } } });
await flush();
state.failedCdns.set([A]);
await flush();
await vi.advanceTimersByTimeAsync(600);
state.failedCdns.set([A, B]); // cdn-b tripped 600ms after cdn-a
await flush();
await vi.advanceTimersByTimeAsync(400); // cdn-a's cooldown lapses (t=1000); cdn-b's (t=1600) not yet
expect(state.failedCdns.get()).toEqual([B]);
await vi.advanceTimersByTimeAsync(600); // cdn-b's cooldown lapses (t=1600)
expect(state.failedCdns.get()).toEqual([]);
reactor.destroy();
});
it('clears failedCdns and pending timers on src unload', async () => {
const state = makeState(resolved());
const reactor = setupFailoverMonitor.setup({ state, config: { failover: { cooldownMs: 1000 } } });
await flush();
state.failedCdns.set([A]);
await flush();
expect(state.failedCdns.get()).toEqual([A]);
state.presentation.set(undefined);
await flush();
expect(state.failedCdns.get()).toBeUndefined();
reactor.destroy();
});
it('exposes a sensible failover default', () => {
expect(DEFAULT_FAILOVER_MONITOR_CONFIG.cooldownMs).toBeGreaterThan(0);
});
});
@@ -12,6 +12,7 @@ import type {
} from '../../../media/types';
import type { BandwidthState } from '../../../network/bandwidth-estimator';
import {
applyConstraints,
applyRules,
type SelectionRule,
type SwitchVideoTrackConfig,
@@ -735,6 +736,43 @@ describe('applyRules', () => {
});
});
// ============================================================================
// applyConstraints — the hard-constraints pre-pass (pure; no signals)
// ============================================================================
describe('applyConstraints', () => {
const track = (id: string) => ({ id });
const all = [track('a'), track('b'), track('c')];
const noDeps = { state: {}, context: {}, config: {} };
const noA: SelectionRule<{ id: string }> = (tracks) => tracks.filter((t) => t.id !== 'a');
const noC: SelectionRule<{ id: string }> = (tracks) => tracks.filter((t) => t.id !== 'c');
it('removes what each constraint excludes (pooled)', () => {
expect(applyConstraints([noA, noC], all, noDeps).map((t) => t.id)).toEqual(['b']);
});
it('is order-independent', () => {
expect(applyConstraints([noA, noC], all, noDeps)).toEqual(applyConstraints([noC, noA], all, noDeps));
});
it('preserves an empty result — no fall-through, unlike applyRules', () => {
const none: SelectionRule<{ id: string }> = () => [];
expect(applyConstraints([none], all, noDeps)).toEqual([]);
});
it('runs every constraint — no early-bail at a single survivor', () => {
const toA: SelectionRule<{ id: string }> = (tracks) => tracks.filter((t) => t.id === 'a');
let laterCalled = false;
const later: SelectionRule<{ id: string }> = (tracks) => {
laterCalled = true;
return tracks;
};
applyConstraints([toA, later], all, noDeps);
expect(laterCalled).toBe(true);
});
});
// ============================================================================
// preferActiveCdn — active-CDN scope (shared by video + audio)
// ============================================================================
@@ -813,8 +851,8 @@ describe('preferActiveCdn (active-CDN scope)', () => {
});
it('applies the scope when cdnPriority arrives after the first pick (composition-order independence)', async () => {
// Guards against the pick depending on `resolveCdnPriority` being composed
// *before* `switchVideoTrack`. The worst case — resolveCdnPriority last — is
// Guards against the pick depending on `deriveCdnPriority` being composed
// *before* `switchVideoTrack`. The worst case — deriveCdnPriority last — is
// equivalent to cdnPriority being written after switch*'s first pick. The
// scope subscribes to cdnPriority even while it's undefined, so a late write
// must re-fire and correct the pick.
@@ -863,3 +901,87 @@ describe('preferActiveCdn (active-CDN scope)', () => {
reactor.destroy();
});
});
// ============================================================================
// excludeFailedCdns — the failover constraint (hard pre-pass) + scope interplay
// ============================================================================
describe('excludeFailedCdns (failover constraint)', () => {
const cdnVideoTrack = (id: string, host: string, bandwidth: number): PartiallyResolvedVideoTrack => ({
type: 'video',
codecs: [],
id,
url: `https://${host}/${id}.m3u8`,
bandwidth,
mimeType: 'video/mp4',
});
const multiCdn = () =>
createPresentation([
cdnVideoTrack('720p-a', 'cdn-a.example.com', 2_400_000),
cdnVideoTrack('720p-b', 'cdn-b.example.com', 2_400_000),
cdnVideoTrack('1080p-a', 'cdn-a.example.com', 4_800_000),
cdnVideoTrack('1080p-b', 'cdn-b.example.com', 4_800_000),
]);
const makeState = (failedCdns?: string[]) => ({
presentation: signal<MaybeResolvedPresentation | undefined>(multiCdn()),
bandwidthState: signal<BandwidthState | undefined>(createBandwidthState(10_000_000)),
selectedVideoTrackId: signal<string | undefined>(undefined),
userVideoTrackSelection: signal<Partial<VideoTrack> | undefined>(undefined),
cdnPriority: signal<string[] | undefined>(['https://cdn-a.example.com', 'https://cdn-b.example.com']),
failedCdns: signal<string[] | undefined>(failedCdns),
});
it('excludes nothing when failedCdns is absent — picks the primary', async () => {
const state = makeState(undefined);
const reactor = switchVideoTrack.setup({ state });
await flush();
expect(state.selectedVideoTrackId.get()).toBe('1080p-a');
reactor.destroy();
});
it('fails over to the next CDN when the primary is in cooldown', async () => {
const state = makeState(['https://cdn-a.example.com']);
const reactor = switchVideoTrack.setup({ state });
await flush();
// cdn-a's tracks are pruned by the constraint, so the scope falls to cdn-b.
expect(state.selectedVideoTrackId.get()).toBe('1080p-b');
reactor.destroy();
});
it('fails over reactively, then returns to the primary on recovery', async () => {
const state = makeState(undefined);
const reactor = switchVideoTrack.setup({ state });
await flush();
expect(state.selectedVideoTrackId.get()).toBe('1080p-a');
// cdn-a enters cooldown → prune → scope falls to cdn-b.
state.failedCdns.set(['https://cdn-a.example.com']);
await flush();
expect(state.selectedVideoTrackId.get()).toBe('1080p-b');
// cdn-a recovers → its tracks reappear → scope snaps back to the primary.
state.failedCdns.set([]);
await flush();
expect(state.selectedVideoTrackId.get()).toBe('1080p-a');
reactor.destroy();
});
it('keeps the prior pick when every CDN is in cooldown (nothing playable)', async () => {
const state = makeState(undefined);
const reactor = switchVideoTrack.setup({ state });
await flush();
expect(state.selectedVideoTrackId.get()).toBe('1080p-a');
// All CDNs cooled down → constraints prune everything → no playable set →
// the effect no-ops, leaving the last pick in place (deferred terminal-state
// modeling).
state.failedCdns.set(['https://cdn-a.example.com', 'https://cdn-b.example.com']);
await flush();
expect(state.selectedVideoTrackId.get()).toBe('1080p-a');
reactor.destroy();
});
});
@@ -4,17 +4,20 @@
* default, react to user intent and algorithmic ranking, and clear it on src
* unload.
*
* Selection runs a small ordered chain of rules over the candidate tracks
* (`applyRules`). Each rule narrows or reorders the list and reads the signals
* it needs at apply time, so the effect subscribes to exactly what the applied
* rules consult. Today the chain is three rules, most authoritative first:
* Selection runs in two stages. First a **hard-constraints pre-pass**
* (`applyConstraints`) prunes the unplayable from the candidate set today the
* failed-CDN constraint (`excludeFailedCdns`, failover cooldown); capability
* probing will join it. Then a small ordered chain of rules (`applyRules`) picks
* among the survivors. Each constraint/rule reads the signals it needs at apply
* time, so the effect subscribes to exactly what was consulted. The chain is
* three rules, most authoritative first:
*
* 1. **user intent** a soft filter on `user*TrackSelection`: narrow to the
* partial-track match; an empty match falls through to the full set.
* 2. **active CDN** a soft filter on `cdnPriority` (`preferActiveCdn`):
* narrow to the highest-priority CDN that still has tracks; an empty match
* falls through. Shared by video and audio, so every type stays on one CDN
* (`resolveCdnPriority` owns the list). No-op for non-redundant sources.
* (`deriveCdnPriority` owns the list). No-op for non-redundant sources.
* 3. **ranking** the terminal sort: `rankByBandwidth`, shared by video and
* audio. Fitting tracks (within the throughput threshold) first, highest
* bitrate first; over-throughput tracks after, least-over first. Hysteresis
@@ -30,17 +33,17 @@
* (canonical cleanup-binds-to-setup per `reactors.md`).
*
* The pick is the head of the chain's result (`applyRules(...)[0]`). Each
* variant supplies its **rule chain** via config; `setupTrackSwitching` owns
* only the lifecycle and runs whatever chain it's given. Both variants today run
* variant supplies its **constraints + rule chain** via config;
* `setupTrackSwitching` owns only the lifecycle and runs what it's given. Both
* variants today run constraints `[excludeFailedCdns]` then rules
* `[filterByUserSelection, preferActiveCdn, rankByBandwidth]`; `switchVideoTrack`
* also accepts ABR tuning config, `switchAudioTrack` takes none.
* also accepts ABR tuning config, `switchAudioTrack` takes none. (The active-CDN
* *scope* is the sticky-pick half of multi-CDN; the failed-CDN *constraint* is
* the failover half prune the cooled-down CDN, the scope falls to the next.)
*
* Deferred (not yet in the chain): a hard-constraints pre-pass (capability
* probing, CDN *failover* excluding a failed CDN's tracks during cooldown)
* gating the candidate set, and audio's preferred-language / default-track
* selection as standing soft-filter rules previously the empty-slot picker,
* dropped in the move to the rule chain. (The active-CDN *scope* above is the
* sticky-pick half of multi-CDN; the failed-CDN constraint is the failover half.)
* Deferred: capability probing as a second constraint; audio's preferred-
* language / default-track selection as standing soft-filter rules (previously
* the empty-slot picker, dropped in the move to the rule chain).
*/
import { type AnySlotMap, defineBehavior } from '../../core/composition/create-composition';
@@ -56,7 +59,7 @@ import {
type PartiallyResolvedVideoTrack,
type VideoTrack,
} from '../../media/types';
import { getCdnId } from '../../media/utils/cdn';
import { getCdnId as defaultGetCdnId, type GetCdnId } from '../../media/utils/cdn';
import { getTracksByType } from '../../media/utils/tracks';
import type { BandwidthConfig, BandwidthState } from '../../network/bandwidth-estimator';
import { DEFAULT_BANDWIDTH_CONFIG, getBandwidthEstimate } from '../../network/bandwidth-estimator';
@@ -91,6 +94,8 @@ export interface SwitchVideoTrackConfig {
quality?: Partial<QualityConfig>;
bandwidth?: Partial<BandwidthConfig>;
initialBandwidth?: number;
/** Override CDN-id derivation (shared by the CDN scope + failover constraint). */
getCdnId?: GetCdnId;
}
/** Default initial-bandwidth value before bandwidth measurements arrive. */
@@ -153,6 +158,31 @@ export function applyRules<T, State, Context, Config>(
return current;
}
/**
* Apply hard constraints to a candidate list the pre-pass that runs before the
* rule chain. A constraint shares a rule's signature but its exclusion is
* *hard*: it removes the unplayable (a codec the environment can't decode, a CDN
* in failover cooldown) and a removed track is never attempted. Unlike
* `applyRules`, this never skips an empty result and never early-bails every
* constraint always applies, and an empty survivor set is a real outcome
* ("nothing playable here"), not a fall-through. Because each constraint only
* removes, the order they run in can't change the result.
*
* @param constraints - Constraints to apply (pooled, order-independent)
* @param tracks - Candidate tracks
* @param deps - The behavior's `{ state, context, config }`, passed to each constraint
* @returns The playable survivors (possibly empty)
*/
export function applyConstraints<T, State, Context, Config>(
constraints: readonly SelectionRule<T, State, Context, Config>[],
tracks: readonly T[],
deps: SelectionRuleDeps<State, Context, Config>
): readonly T[] {
let current = tracks;
for (const constraint of constraints) current = constraint(current, deps);
return current;
}
// ============================================================================
// Specialization helper
//
@@ -216,16 +246,19 @@ type TrackSwitchingStateMap<S extends SelectionKey> = {
/**
* Config `setupTrackSwitching` itself reads its own wiring: which selection
* slot to write and clear (`selectionKey`), how to enumerate candidate tracks
* (`getTracks`), and the **rule chain** to run (`rules`). Rule-specific config
* is deliberately absent each rule declares the fields it reads as *optional*
* on its own config view (`UserSelectionConfig`, `BandwidthRankerConfig`), so
* the behavior never enumerates a rule's config. The variant builds the
* concrete config as this base plus whatever its chain's rules consult; it
* flows through untouched as the `C` type param on `setupTrackSwitching`.
* (`getTracks`), the optional **hard-constraints pre-pass** (`constraints`,
* applied before the chain to prune the unplayable), and the **rule chain** to
* run (`rules`). Rule-/constraint-specific config is deliberately absent each
* declares the fields it reads as *optional* on its own config view
* (`UserSelectionConfig`, `BandwidthRankerConfig`), so the behavior never
* enumerates them. The variant builds the concrete config as this base plus
* whatever its chain consults; it flows through untouched as the `C` type param
* on `setupTrackSwitching`.
*/
interface TrackSwitchingConfig<S extends SelectionKey, T extends SwitchableTrack> {
selectionKey: S;
getTracks: (presentation: MaybeResolvedPresentation) => readonly T[];
constraints?: readonly SelectionRule<T, TrackSwitchingStateMap<S>, AnySlotMap, TrackSwitchingConfig<S, T>>[];
rules: readonly SelectionRule<T, TrackSwitchingStateMap<S>, AnySlotMap, TrackSwitchingConfig<S, T>>[];
}
@@ -277,7 +310,7 @@ type BandwidthRankerConfig<S extends SelectionKey, T extends SwitchableTrack> =
/**
* State the active-CDN scope reads: the lifecycle map plus an *optional*
* `cdnPriority` the manifest-ordered CDN list (most-preferred first). The
* signal exists only when the composition includes `resolveCdnPriority` (which
* signal exists only when the composition includes `deriveCdnPriority` (which
* materializes + owns it); the scope reads it defensively and passes through
* when it's absent (no CDN preference).
*/
@@ -285,6 +318,27 @@ type CdnScopeStateMap<S extends SelectionKey> = TrackSwitchingStateMap<S> & {
cdnPriority?: ReadonlySignal<string[] | undefined>;
};
/**
* State the failed-CDN constraint reads: the lifecycle map plus an *optional*
* `failedCdns` the CDN ids currently in failover cooldown. The signal exists
* only when the composition includes a failover monitor (or an external driver); the
* constraint reads it defensively and excludes nothing when it's absent.
*/
type CdnConstraintStateMap<S extends SelectionKey> = TrackSwitchingStateMap<S> & {
failedCdns?: ReadonlySignal<string[] | undefined>;
};
/**
* Config the CDN rules read: the base config plus an *optional* `getCdnId`
* override. Both `excludeFailedCdns` and `preferActiveCdn` derive a track's CDN
* from its URL; the override must be the *same* one `deriveCdnPriority` and the
* failover trip use, or the keys stop matching. Optional defaults to the
* origin-based `getCdnId`, so the base config (without it) stays assignable.
*/
type CdnRuleConfig<S extends SelectionKey, T extends SwitchableTrack> = TrackSwitchingConfig<S, T> & {
getCdnId?: GetCdnId;
};
type VideoTrackCandidate = PartiallyResolvedVideoTrack | VideoTrack;
type AudioTrackCandidate = PartiallyResolvedAudioTrack | AudioTrack;
@@ -310,9 +364,32 @@ function filterByUserSelection<S extends SelectionKey, U extends UserSelectionKe
return filter ? tracks.filter((track) => matchesPartialTrack(track, filter)) : tracks;
}
/**
* Failed-CDN constraint a *hard* filter (constraints pre-pass), shared by
* video and audio. Removes tracks served from a CDN currently in failover
* cooldown (`failedCdns`, written by the failover monitor). Removed tracks are never
* attempted; the scope then narrows to the next surviving CDN in `cdnPriority`,
* and snaps back to the primary once it leaves cooldown.
*
* Passes everything through when there's no `failedCdns` signal/value. When it
* prunes *every* track (all CDNs cooled down), the empty result is preserved
* (per `applyConstraints`) "nothing playable," which today leaves the prior
* pick in place.
*/
function excludeFailedCdns<S extends SelectionKey, T extends SwitchableTrack>(
tracks: readonly T[],
{ state, config }: SelectionRuleDeps<CdnConstraintStateMap<S>, AnySlotMap, CdnRuleConfig<S, T>>
): readonly T[] {
const failed = state.failedCdns?.get();
if (!failed?.length) return tracks;
const getCdnId = config.getCdnId ?? defaultGetCdnId;
const failedSet = new Set(failed);
return tracks.filter((track) => !failedSet.has(getCdnId(track.url)));
}
/**
* Active-CDN scope a soft filter, shared by video and audio. Narrows to the
* highest-priority CDN in `cdnPriority` (owned by `resolveCdnPriority`) that
* highest-priority CDN in `cdnPriority` (owned by `deriveCdnPriority`) that
* still has tracks, so every track type stays on one CDN. A redundant-streams
* source lists the same renditions on multiple hosts; this keeps the pick on one
* host rather than letting the ranker drift across them.
@@ -327,15 +404,17 @@ function filterByUserSelection<S extends SelectionKey, U extends UserSelectionKe
* (no preference) or when nothing matches (`applyRules` skips an empty result).
* Non-redundant sources have one CDN, so the narrow is a no-op.
*
* The CDN-id derivation (`getCdnId`, origin-based) is hardcoded for now; a
* consumer-configurable derivation can move onto a rule config view later.
* The CDN-id derivation defaults to origin-based `getCdnId`, overridable via the
* `getCdnId` config it must match the one `deriveCdnPriority` used to build
* `cdnPriority`, or no track's CDN would ever equal an entry.
*/
function preferActiveCdn<S extends SelectionKey, T extends SwitchableTrack>(
tracks: readonly T[],
{ state }: SelectionRuleDeps<CdnScopeStateMap<S>, AnySlotMap, TrackSwitchingConfig<S, T>>
{ state, config }: SelectionRuleDeps<CdnScopeStateMap<S>, AnySlotMap, CdnRuleConfig<S, T>>
): readonly T[] {
const cdnPriority = state.cdnPriority?.get();
if (!cdnPriority?.length) return tracks;
const getCdnId = config.getCdnId ?? defaultGetCdnId;
for (const cdn of cdnPriority) {
const tracksUsingCdn = tracks.filter((track) => getCdnId(track.url) === cdn);
if (tracksUsingCdn.length) return tracksUsingCdn;
@@ -416,15 +495,13 @@ function setupTrackSwitching<
);
// The playable candidate set — the tracks the rule chain gets to pick from,
// derived *outside* the reaction. This is the seam a future hard-constraints
// pre-pass (capability probing, CDN failover) occupies: it would narrow these
// tracks before the chain runs —
// isResolvedPresentation(p) ? applyConstraints(constraints, getTracks(p), deps) : []
// — and because it's a `computed`, the constraints' own signal reads are
// tracked here. The effect reads it with `.get()`, so when the playable set
// changes — a new source, or a *dynamic* constraint like a CDN entering
// cooldown — the effect re-picks. Today it's just the type's tracks while a
// presentation is resolved.
// derived *outside* the reaction. The hard-constraints pre-pass (capability
// probing, CDN-failover cooldown) narrows the type's tracks before the chain
// runs. Because this is a `computed`, a constraint's own signal reads (e.g.
// `cdnHealth`) are tracked here, so when the playable set changes — a new
// source, or a *dynamic* constraint like a CDN entering cooldown — the effect
// re-picks. With no constraints configured this is just the type's tracks
// while a presentation is resolved.
//
// The `equals` gates notification on the *set of track ids*, not array
// identity: a live playlist refresh swaps in a new presentation object with
@@ -436,7 +513,8 @@ function setupTrackSwitching<
const candidateSet = computed<readonly T[]>(
() => {
const presentation = state.presentation.get();
return isResolvedPresentation(presentation) ? getTracks(presentation) : [];
if (!isResolvedPresentation(presentation)) return [];
return applyConstraints(config.constraints ?? [], getTracks(presentation), deps);
},
{ equals: (a, b) => a.length === b.length && a.every((track) => b.some((other) => other.id === track.id)) }
);
@@ -524,6 +602,7 @@ export const switchVideoTrack = defineBehavior({
selectionKey: 'selectedVideoTrackId',
userSelectionKey: 'userVideoTrackSelection',
getTracks: (presentation) => getTracksByType(presentation, 'video') as readonly VideoTrackCandidate[],
constraints: [excludeFailedCdns],
rules: [filterByUserSelection, preferActiveCdn, rankByBandwidth],
},
}),
@@ -549,14 +628,32 @@ export const switchVideoTrack = defineBehavior({
export const switchAudioTrack = defineBehavior({
stateKeys: ['presentation', 'selectedAudioTrackId'],
contextKeys: [],
setup: ({ state, ...otherProps }: { state: TrackSwitchingStateMap<'selectedAudioTrackId'> }) =>
setup: ({
state,
config,
...otherProps
}: {
state: TrackSwitchingStateMap<'selectedAudioTrackId'>;
// Shares the video config shape so the engine config spreads through (CDN
// derivation + any future cross-cutting fields).
config?: SwitchVideoTrackConfig;
}) =>
setupTrackSwitching({
...otherProps,
state,
config: {
// Spread engine config so cross-cutting fields (`getCdnId`, future shared
// tuning) flow through like they do for video, then override the per-type
// wiring. Video-only ABR tuning (`quality`/`bandwidth`/`initialBandwidth`)
// rides along into the shared `rankByBandwidth` too; harmless since audio
// has no `bandwidthState` to act on it and the ranker always yields a pick.
// FOLLOW-UP: a shared config type for the genuinely cross-cutting fields
// would keep video-only tuning out of audio entirely (CJP).
...config,
selectionKey: 'selectedAudioTrackId',
userSelectionKey: 'userAudioTrackSelection',
getTracks: (presentation) => getTracksByType(presentation, 'audio') as readonly AudioTrackCandidate[],
constraints: [excludeFailedCdns],
rules: [filterByUserSelection, preferActiveCdn, rankByBandwidth],
},
}),
@@ -11,6 +11,7 @@
{ "path": "../../core" },
{ "path": "../../network" },
{ "path": "../../media" },
{ "path": "../primitives" },
{ "path": "../actors" }
],
"include": ["./*.ts", "./tests/**/*.ts"],
@@ -9,6 +9,7 @@ import type { BackBufferConfig } from '../../../media/buffer/back-buffer';
import type { ForwardBufferConfig } from '../../../media/buffer/forward-buffer';
import { parseMultivariantPlaylist } from '../../../media/hls/parse-multivariant';
import type { AudioTrack, MaybeResolvedPresentation } from '../../../media/types';
import type { GetCdnId } from '../../../media/utils/cdn';
import { getResolvedSelectedTrackDuration } from '../../../media/utils/track-selection';
import type { SegmentLoaderActor } from '../../actors/dom/segment-loader';
import type { SourceBufferActor } from '../../actors/dom/source-buffer';
@@ -16,6 +17,7 @@ import {
calculatePresentationDuration,
type PresentationDurationResolver,
} from '../../behaviors/calculate-presentation-duration';
import { deriveCdnPriority } from '../../behaviors/derive-cdn-priority';
import { endOfStream } from '../../behaviors/dom/end-of-stream';
import { loadAudioSegments } from '../../behaviors/dom/load-segments';
import { setupAudioBufferActors } from '../../behaviors/dom/setup-buffer-actors';
@@ -23,9 +25,9 @@ import { setupMediaSource } from '../../behaviors/dom/setup-mediasource';
import { trackCurrentTime } from '../../behaviors/dom/track-current-time';
import { trackLoadTriggers } from '../../behaviors/dom/track-load-triggers';
import { updateMediaSourceDuration } from '../../behaviors/dom/update-mediasource-duration';
import { resolveCdnPriority } from '../../behaviors/resolve-cdn-priority';
import { type ParsePresentation, resolvePresentation } from '../../behaviors/resolve-presentation';
import { resolveAudioTrack } from '../../behaviors/resolve-track';
import { type FailoverMonitorConfig, setupFailoverMonitor } from '../../behaviors/setup-failover-monitor';
import { syncPreload } from '../../behaviors/sync-preload';
import { switchAudioTrack } from '../../behaviors/track-switching';
@@ -54,11 +56,17 @@ export interface SimpleHlsAudioOnlyEngineState {
userAudioTrackSelection?: Partial<AudioTrack>;
/**
* The CDNs the source is served from, in manifest priority order (mirrors
* HLS content steering's `PATHWAY-PRIORITY`). Owned by `resolveCdnPriority`,
* HLS content steering's `PATHWAY-PRIORITY`). Owned by `deriveCdnPriority`,
* read by `track-switching`'s `preferActiveCdn` scope. Only meaningful for
* redundant-stream sources; a single-CDN source has one entry.
*/
cdnPriority?: string[];
/**
* CDN ids currently in failover cooldown read by `track-switching`'s
* `excludeFailedCdns` constraint, which prunes their tracks so the active-CDN
* scope falls to the next CDN. Empty / absent means all CDNs are eligible.
*/
failedCdns?: string[];
currentTime?: number;
loadActivated?: boolean;
}
@@ -94,14 +102,24 @@ export interface SimpleHlsAudioOnlyEngineConfig
parsePresentation?: ParsePresentation;
forwardBuffer?: Partial<ForwardBufferConfig>;
backBuffer?: Partial<BackBufferConfig>;
/** Multi-CDN failover monitor tuning. Defaults: `DEFAULT_FAILOVER_MONITOR_CONFIG`. */
failover?: Partial<FailoverMonitorConfig>;
/**
* Derive a CDN grouping key from a track URL (used by `cdnPriority`, the
* failover trip, and the track-switching CDN rules one function read by all).
* Defaults to the URL origin; override to key on e.g. Mux's `cdn=` param.
*/
getCdnId?: GetCdnId;
}
// ============================================================================
// Audio-Only HLS Playback Engine
// ============================================================================
// Materializes the consumer-input slot `userAudioTrackSelection` (only read by
// switchAudioTrack, produced by no behavior) in addition to forwarding refs.
// Materializes input slots no composed behavior produces — `userAudioTrackSelection`
// (switchAudioTrack only reads it) — in addition to forwarding refs. `failedCdns`
// is owned by `setupFailoverMonitor`, so it's already materialized and reachable
// on the `onSignalsReady` refs without being listed here.
const shareSignals = makeShareSignals<SimpleHlsAudioOnlyEngineState, SimpleHlsAudioOnlyEngineContext>([
'userAudioTrackSelection',
]);
@@ -160,7 +178,12 @@ export function createHlsAudioOnlyEngine(
// not load-bearing here today. It earns its place for forward-consistency
// with the default engine and for future failover / steering, where the
// active CDN changes dynamically (and selection stays reactive either way).
resolveCdnPriority,
deriveCdnPriority,
// CDN failover cooldown: watches `failedCdns` (tripped directly by audio
// track resolution on a failed media-playlist fetch) and removes each CDN
// once its cooldown lapses.
setupFailoverMonitor,
// Audio track selection — slot owner with filter reactivity.
// Mid-stream flush on language switch is handled in segment-loader's
@@ -16,6 +16,7 @@ import {
} from '../../../media/dom/text/text-track-slots';
import { parseMultivariantPlaylist } from '../../../media/hls/parse-multivariant';
import type { AudioTrack, MaybeResolvedPresentation, VideoTrack } from '../../../media/types';
import type { GetCdnId } from '../../../media/utils/cdn';
import { getResolvedSelectedTrackDuration } from '../../../media/utils/track-selection';
import type { BandwidthConfig, BandwidthState } from '../../../network/bandwidth-estimator';
import type { SegmentLoaderActor } from '../../actors/dom/segment-loader';
@@ -26,6 +27,7 @@ import {
calculatePresentationDuration,
type PresentationDurationResolver,
} from '../../behaviors/calculate-presentation-duration';
import { deriveCdnPriority } from '../../behaviors/derive-cdn-priority';
import { endOfStream } from '../../behaviors/dom/end-of-stream';
import { loadAudioSegments, loadTextTrackSegments, loadVideoSegments } from '../../behaviors/dom/load-segments';
import { setupAudioBufferActors, setupVideoBufferActors } from '../../behaviors/dom/setup-buffer-actors';
@@ -35,10 +37,10 @@ import { syncTextTracks } from '../../behaviors/dom/sync-text-tracks';
import { trackCurrentTime } from '../../behaviors/dom/track-current-time';
import { trackLoadTriggers } from '../../behaviors/dom/track-load-triggers';
import { updateMediaSourceDuration } from '../../behaviors/dom/update-mediasource-duration';
import { resolveCdnPriority } from '../../behaviors/resolve-cdn-priority';
import { type ParsePresentation, resolvePresentation } from '../../behaviors/resolve-presentation';
import { resolveAudioTrack, resolveTextTrack, resolveVideoTrack } from '../../behaviors/resolve-track';
import { selectTextTrack } from '../../behaviors/select-tracks';
import { type FailoverMonitorConfig, setupFailoverMonitor } from '../../behaviors/setup-failover-monitor';
import { syncPreload } from '../../behaviors/sync-preload';
import { switchAudioTrack, switchVideoTrack } from '../../behaviors/track-switching';
@@ -75,13 +77,21 @@ export interface SimpleHlsEngineState {
/**
* The CDNs the source is served from (track-URL origins), in manifest
* priority order most-preferred first (mirrors HLS content steering's
* `PATHWAY-PRIORITY`). Owned by `resolveCdnPriority`, read by
* `PATHWAY-PRIORITY`). Owned by `deriveCdnPriority`, read by
* `track-switching`'s `preferActiveCdn` scope, which narrows to the
* highest-priority CDN with surviving tracks so video / audio / text stay on
* one host. Only meaningful for redundant-stream sources; a single-CDN source
* has one entry.
*/
cdnPriority?: string[];
/**
* CDN ids (origins) currently in failover cooldown written by the CDN
* monitor when a host fails too often, read by `track-switching`'s
* `excludeFailedCdns` hard constraint, which prunes their tracks so the
* active-CDN scope falls to the next CDN in `cdnPriority`. Empty / absent
* means all CDNs are eligible.
*/
failedCdns?: string[];
currentTime?: number;
loadActivated?: boolean;
}
@@ -196,6 +206,21 @@ export interface SimpleHlsEngineConfig extends ShareSignalsConfig<SimpleHlsEngin
* ratio gating ABR upgrades. Defaults: `DEFAULT_QUALITY_CONFIG` (0.85 / 1.15).
*/
quality?: Partial<QualityConfig>;
/**
* Multi-CDN failover monitor tuning. `cooldownMs` is how long a CDN stays
* excluded after a failed fetch trips it. Defaults:
* `DEFAULT_FAILOVER_MONITOR_CONFIG` (300s). Only meaningful for redundant-stream
* sources.
*/
failover?: Partial<FailoverMonitorConfig>;
/**
* How to derive a CDN grouping key from a track URL used to build
* `cdnPriority`, to record the failover trip in `failedCdns`, and by the
* track-switching CDN scope + failover constraint. One function, read by all of
* them, so the keys stay comparable. Defaults to the URL origin; override to
* key on something else (e.g. Mux's `cdn=` query param).
*/
getCdnId?: GetCdnId;
}
// ============================================================================
@@ -205,9 +230,11 @@ export interface SimpleHlsEngineConfig extends ShareSignalsConfig<SimpleHlsEngin
/**
* Generic `shareSignals` instantiated against the HLS engine's full state
* and context captures composition signal refs into the consumer's
* `onSignalsReady` callback at setup time, and materializes the consumer-input
* slots (`user*TrackSelection`) that no behavior produces: the track-switching
* behaviors only *read* them, so shareSignals owns bringing them into existence.
* `onSignalsReady` callback at setup time, and materializes input slots that no
* composed behavior produces: `user*TrackSelection` (track-switching only reads
* them). `failedCdns` is owned by `setupFailoverMonitor`, so it's already
* materialized and reachable on the `onSignalsReady` refs without being listed
* here.
*/
const shareSignals = makeShareSignals<SimpleHlsEngineState, SimpleHlsEngineContext>([
'userVideoTrackSelection',
@@ -273,7 +300,12 @@ export function createSimpleHlsEngine(
// media-playlist fetch to the wrong CDN before correcting. Symmetric
// redundant streams (the norm) never hit it — the first-listed CDN is
// already the primary we'd pick anyway.
resolveCdnPriority,
deriveCdnPriority,
// CDN failover cooldown: owns the expiry half of failover — watches
// `failedCdns` (tripped directly by track resolution on a failed
// media-playlist fetch) and removes each CDN once its cooldown lapses.
setupFailoverMonitor,
// Track selection (reads config for initial preferences).
// Video selection lives in switchVideoTrack (composed below);
@@ -133,7 +133,7 @@ describe('createSimpleHlsEngine', () => {
});
it('keeps audio on the same CDN as video even when the audio rendition order differs', async () => {
// Order-effect guard: `resolveCdnPriority` derives the list from track order,
// Order-effect guard: `deriveCdnPriority` derives the list from track order,
// so a same-ordered source can't distinguish "scope applied" from "scope is
// a no-op". This source is doubly adversarial to the desired result: the
// audio selection set comes BEFORE video in the manifest, and within it the
@@ -210,6 +210,176 @@ describe('createSimpleHlsEngine', () => {
engine.destroy();
});
it('fails over video and audio to the next CDN when one is marked failed', async () => {
const flush = () => Promise.resolve().then(() => Promise.resolve());
const engine = createSimpleHlsEngine();
const videoTrack = (id: string, host: string): PartiallyResolvedVideoTrack => ({
type: 'video',
id,
codecs: [],
url: `https://${host}/${id}.m3u8`,
bandwidth: 2_400_000,
mimeType: 'video/mp4',
});
const audioTrack = (id: string, host: string): PartiallyResolvedAudioTrack => ({
type: 'audio',
id,
codecs: ['mp4a.40.2'],
url: `https://${host}/${id}.m3u8`,
bandwidth: 128_000,
mimeType: 'audio/mp4',
groupId: 'audio',
name: id,
sampleRate: 48_000,
channels: 2,
});
engine.state.presentation.set({
id: 'pres-failover',
url: 'https://cdn-a.example.com/master.m3u8',
startTime: 0,
selectionSets: [
{
id: 'v',
type: 'video',
switchingSets: [
{
id: 'vs',
type: 'video',
tracks: [videoTrack('vid-a', 'cdn-a.example.com'), videoTrack('vid-b', 'cdn-b.example.com')],
},
],
},
{
id: 'a',
type: 'audio',
switchingSets: [
{
id: 'as',
type: 'audio',
tracks: [audioTrack('aud-a', 'cdn-a.example.com'), audioTrack('aud-b', 'cdn-b.example.com')],
},
],
},
],
} as Presentation);
await flush();
// Primary CDN initially.
expect(engine.state.selectedVideoTrackId.get()).toBe('vid-a');
expect(engine.state.selectedAudioTrackId.get()).toBe('aud-a');
// Mark cdn-a failed → both types fail over to cdn-b coherently.
engine.state.failedCdns.set(['https://cdn-a.example.com']);
await flush();
expect(engine.state.selectedVideoTrackId.get()).toBe('vid-b');
expect(engine.state.selectedAudioTrackId.get()).toBe('aud-b');
// cdn-a recovers → both return to the primary.
engine.state.failedCdns.set([]);
await flush();
expect(engine.state.selectedVideoTrackId.get()).toBe('vid-a');
expect(engine.state.selectedAudioTrackId.get()).toBe('aud-a');
engine.destroy();
});
it('auto-fails-over when a CDN fetch fails (monitor trips, failedCdns set)', async () => {
const engine = createSimpleHlsEngine({ failover: { cooldownMs: 60_000 } });
// cdn-a is down (media-playlist fetch rejects); cdn-b serves a valid playlist.
globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => {
const url = typeof input === 'string' ? input : String((input as Request).url ?? input);
if (url.includes('cdn-a')) throw new TypeError('cdn-a unreachable');
return new Response('#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXTINF:10.0,\nseg-1.m4s\n#EXT-X-ENDLIST');
}) as typeof fetch;
const videoTrack = (id: string, host: string): PartiallyResolvedVideoTrack => ({
type: 'video',
id,
codecs: [],
url: `https://${host}/${id}.m3u8`,
bandwidth: 2_400_000,
mimeType: 'video/mp4',
});
engine.state.presentation.set({
id: 'pres-failover',
url: 'https://cdn-a.example.com/master.m3u8',
startTime: 0,
selectionSets: [
{
id: 'v',
type: 'video',
switchingSets: [
{
id: 'vs',
type: 'video',
tracks: [videoTrack('vid-a', 'cdn-a.example.com'), videoTrack('vid-b', 'cdn-b.example.com')],
},
],
},
],
} as Presentation);
// The primary (cdn-a) is picked first, its media-playlist fetch fails, the
// monitor trips it, the constraint prunes it, and the scope fails over to
// cdn-b — all without any external failedCdns write.
await vi.waitFor(() => {
expect(engine.state.failedCdns.get()).toEqual(['https://cdn-a.example.com']);
expect(engine.state.selectedVideoTrackId.get()).toBe('vid-b');
});
engine.destroy();
});
it('honors a custom getCdnId across cdnPriority, the trip, and the constraint/scope', async () => {
// Key CDNs on the `cdn=` query param instead of origin. Both variants share a
// host, so origin-based identity would see ONE CDN (no redundancy); the
// custom resolver must be respected at every site for failover to work.
const getCdnId = (url: string) => new URL(url).searchParams.get('cdn') ?? url;
const engine = createSimpleHlsEngine({ getCdnId, failover: { cooldownMs: 60_000 } });
globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => {
const url = typeof input === 'string' ? input : String((input as Request).url ?? input);
if (url.includes('cdn=a')) throw new TypeError('cdn-a unreachable');
return new Response('#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXTINF:10.0,\nseg-1.m4s\n#EXT-X-ENDLIST');
}) as typeof fetch;
const videoTrack = (id: string, cdn: string): PartiallyResolvedVideoTrack => ({
type: 'video',
id,
codecs: [],
url: `https://cdn.example.com/${id}.m3u8?cdn=${cdn}`,
bandwidth: 2_400_000,
mimeType: 'video/mp4',
});
engine.state.presentation.set({
id: 'pres-custom-cdn',
url: 'https://cdn.example.com/master.m3u8',
startTime: 0,
selectionSets: [
{
id: 'v',
type: 'video',
switchingSets: [{ id: 'vs', type: 'video', tracks: [videoTrack('vid-a', 'a'), videoTrack('vid-b', 'b')] }],
},
],
} as Presentation);
await vi.waitFor(() => {
// deriveCdnPriority keyed on the param (not origin → not a single CDN).
expect(engine.state.cdnPriority.get()).toEqual(['a', 'b']);
// The trip recorded the param key, and the constraint + scope failed over.
expect(engine.state.failedCdns.get()).toEqual(['a']);
expect(engine.state.selectedVideoTrackId.get()).toBe('vid-b');
});
engine.destroy();
});
it('allows patching state and owners from outside', async () => {
const engine = createSimpleHlsEngine();
@@ -0,0 +1,87 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { isResolvedTrack, type MaybeResolvedPresentation } from '../../../../media/types';
import { createSimpleHlsEngine } from '../engine';
// Live smoke test for multi-CDN failover against a real Mux `redundant_streams`
// source. It hits the network (the manifest + the surviving CDN are fetched for
// real), so it's gated behind VITE_FAILOVER_SMOKE and skipped in the default run.
//
// VITE_FAILOVER_SMOKE=1 pnpm -F @videojs/spf test src/playback/engines/hls/tests/failover-smoke.test.ts
//
// We can't make a real Mux CDN drop requests, so the failure is "hacked": a
// fetch wrapper rejects every request to the primary origin while letting the
// manifest and the backup origin hit the real network.
const SMOKE = (import.meta as unknown as { env?: Record<string, string | undefined> }).env?.VITE_FAILOVER_SMOKE;
const REDUNDANT_URL = 'https://stream.mux.com/s41JYeqIpBMBzE4OzxDyGR2yrp2hD1CQ6gJN9SlVGDQ.m3u8?redundant_streams=true';
// This asset duplicates every variant across two origins; edgemv is listed
// first, so it resolves to cdnPriority[0] and is the one we block.
const PRIMARY = 'edgemv.mux.com';
const BACKUP = 'fastly.mux.com';
const hostOf = (url: string): string => new URL(url).host;
function selectedVideoTrack(presentation: MaybeResolvedPresentation | undefined, id: string | undefined) {
if (!presentation || !id) return undefined;
for (const set of presentation.selectionSets ?? []) {
for (const sw of set.switchingSets) {
const track = sw.tracks.find((t) => t.id === id);
if (track) return track;
}
}
return undefined;
}
describe.skipIf(!SMOKE)('multi-CDN failover (live smoke)', () => {
let realFetch: typeof globalThis.fetch;
afterEach(() => {
globalThis.fetch = realFetch;
});
it('fails over to the backup CDN when the primary is unreachable, then recovers', async () => {
realFetch = globalThis.fetch;
let blockPrimary = true;
globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : (input as Request).url;
if (blockPrimary && url.includes(PRIMARY)) return Promise.reject(new TypeError('blocked (smoke)'));
return realFetch(input as RequestInfo, init);
}) as typeof fetch;
const engine = createSimpleHlsEngine({ failover: { cooldownMs: 4000 } });
engine.state.presentation.set({ url: REDUNDANT_URL } as MaybeResolvedPresentation);
// The primary is picked first, its media-playlist fetch fails, the trip
// lands in failedCdns, the constraint prunes it, and the selected video
// track resolves on the backup CDN.
await vi.waitFor(
() => {
expect(engine.state.cdnPriority.get()?.length).toBe(2);
expect(engine.state.failedCdns.get()?.some((cdn) => cdn.includes(PRIMARY))).toBe(true);
const track = selectedVideoTrack(engine.state.presentation.get(), engine.state.selectedVideoTrackId.get());
expect(track).toBeDefined();
expect(hostOf(track!.url)).toContain(BACKUP);
expect(isResolvedTrack(track!)).toBe(true);
},
{ timeout: 20_000, interval: 250 }
);
// Recovery: unblock the primary. Once its cooldown lapses it leaves
// failedCdns and (being cdnPriority[0]) is preferred again — selection flips
// back and the primary playlist now resolves for real.
blockPrimary = false;
await vi.waitFor(
() => {
expect(engine.state.failedCdns.get()?.some((cdn) => cdn.includes(PRIMARY))).toBe(false);
const track = selectedVideoTrack(engine.state.presentation.get(), engine.state.selectedVideoTrackId.get());
expect(track).toBeDefined();
expect(hostOf(track!.url)).toContain(PRIMARY);
expect(isResolvedTrack(track!)).toBe(true);
},
{ timeout: 20_000, interval: 250 }
);
await engine.destroy();
}, 60_000);
});
@@ -0,0 +1,63 @@
import { type ReadonlySignal, type Signal, update } from '../../core/signals/primitives';
import type { MaybeResolvedPresentation } from '../../media/types';
import { addFailedCdn, getCdnId as defaultGetCdnId, type GetCdnId } from '../../media/utils/cdn';
import { findTrackById } from '../../media/utils/tracks';
import type { FetchOptions, Resource } from '../../network/fetch';
type SelectedTrackKey = 'selectedVideoTrackId' | 'selectedAudioTrackId' | 'selectedTextTrackId';
/**
* State a failover-decorated fetch reads: the presentation, the per-type
* selected-track slot, and the failover monitor's `failedCdns`.
*
* `failedCdns` is *optional* the failover monitor owns that slot, so a
* behavior's narrow state is assignable here without the behavior declaring it
* (and the intersection shares keys, so it isn't a weak type). When no monitor
* is composed the slot is absent and tracking no-ops.
*/
type FailoverState<K extends SelectedTrackKey> = {
presentation: ReadonlySignal<MaybeResolvedPresentation | undefined>;
failedCdns?: Signal<string[] | undefined>;
} & { [P in K]: ReadonlySignal<string | undefined> };
/** Any `Resource`-addressable fetch — both `FetchText` and `FetchBytes` qualify. */
type FailoverableFetch = (addressable: Resource, options?: FetchOptions) => Promise<unknown>;
/**
* Decorate a fetch so a failed request trips the **selected track's** CDN into
* `failedCdns`. The decorated fetch's type is preserved, so this wraps both
* `resolve-track`'s playlist `FetchText` and the segment loaders' `FetchBytes`.
*
* The CDN id comes from the selected track's media-playlist URL, never the
* failed addressable: a segment URL resolves relative to its playlist and, per
* RFC 3986, drops the playlist's query string (`…/r.m3u8?cdn=fastly` `…/0.ts`),
* so a query-keyed `getCdnId` (e.g. Mux's `cdn=`) keyed on it would derive an id
* that never matches the ones `deriveCdnPriority` / track-switching build from
* `track.url`. The in-flight fetch belongs to the selected track a source or
* track switch aborts it, and aborts don't trip so the selected track is the
* right CDN to fail over. For `resolve-track` the resolving track *is* the
* selected track, so this is identical to keying on its addressable.
*
* No-op when no failover monitor is composed (it owns the signal) or the
* selected track can't be located.
*/
export function failoverFetch<K extends SelectedTrackKey, Fetch extends FailoverableFetch>(
baseFetch: Fetch,
state: FailoverState<K>,
config: { selectedKey: K; getCdnId?: GetCdnId }
): Fetch {
const getCdnId = config.getCdnId ?? defaultGetCdnId;
return (async (addressable: Resource, options?: FetchOptions) => {
try {
return await baseFetch(addressable, options);
} catch (error) {
if (!options?.signal?.aborted && state.failedCdns) {
const presentation = state.presentation.get();
const trackId = state[config.selectedKey].get();
const track = presentation && trackId ? findTrackById(presentation, trackId) : undefined;
if (track) update(state.failedCdns, (cdns) => addFailedCdn(cdns, getCdnId(track.url)));
}
throw error;
}
}) as Fetch;
}
@@ -0,0 +1,73 @@
import { describe, expect, it } from 'vitest';
import { signal } from '../../../core/signals/primitives';
import type { MaybeResolvedPresentation } from '../../../media/types';
import type { FetchText } from '../../../network/fetch';
import { failoverFetch } from '../failover-fetch';
const presentationWithVideo = (url: string): MaybeResolvedPresentation => ({
url: 'https://cdn-a.example.com/master.m3u8',
selectionSets: [
{
id: 'video-set',
type: 'video',
switchingSets: [{ id: 'sw', type: 'video', tracks: [{ id: 'v0', type: 'video', url, bandwidth: 0 }] }],
},
] as MaybeResolvedPresentation['selectionSets'],
});
// A query-keyed getCdnId (e.g. Mux's `cdn=`), falling back to origin when the
// param is absent — as it is on segment URLs after relative resolution.
const byCdnParam = (url: string) => new URL(url).searchParams.get('cdn') ?? new URL(url).origin;
const makeState = (presentation: MaybeResolvedPresentation, selectedId: string | undefined) => ({
presentation: signal<MaybeResolvedPresentation | undefined>(presentation),
selectedVideoTrackId: signal<string | undefined>(selectedId),
failedCdns: signal<string[] | undefined>(undefined),
});
const reject: FetchText = async () => {
throw new Error('boom');
};
// The segment URL has dropped the playlist's `?cdn=` param during relative
// resolution, so it's not a valid CDN-identity source.
const segment = { url: 'https://cdn-a.example.com/0.ts' };
describe('failoverFetch', () => {
it('trips the selected track CDN — not the failed addressable — on a failed fetch', async () => {
const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8?cdn=fastly'), 'v0');
const fetch = failoverFetch(reject, state, { selectedKey: 'selectedVideoTrackId', getCdnId: byCdnParam });
await expect(fetch(segment)).rejects.toThrow('boom');
// Keyed on the track URL (`cdn=fastly`), not the param-less segment URL.
expect(state.failedCdns.get()).toEqual(['fastly']);
expect(byCdnParam(segment.url)).not.toBe('fastly');
});
it('does not trip on an aborted fetch', async () => {
const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8?cdn=fastly'), 'v0');
const fetch = failoverFetch(reject, state, { selectedKey: 'selectedVideoTrackId', getCdnId: byCdnParam });
const controller = new AbortController();
controller.abort();
await expect(fetch(segment, { signal: controller.signal })).rejects.toThrow();
expect(state.failedCdns.get()).toBeUndefined();
});
it('no-ops when the selected track cannot be located', async () => {
const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8?cdn=fastly'), 'missing');
const fetch = failoverFetch(reject, state, { selectedKey: 'selectedVideoTrackId', getCdnId: byCdnParam });
await expect(fetch(segment)).rejects.toThrow();
expect(state.failedCdns.get()).toBeUndefined();
});
it('passes a successful fetch through unchanged', async () => {
const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8'), 'v0');
const ok: FetchText = async () => 'body';
const fetch = failoverFetch(ok, state, { selectedKey: 'selectedVideoTrackId' });
await expect(fetch(segment)).resolves.toBe('body');
expect(state.failedCdns.get()).toBeUndefined();
});
});
@@ -0,0 +1,16 @@
{
"extends": "../../../../../tsconfig.base.json",
"compilerOptions": {
"composite": true,
"lib": ["ES2022", "WebWorker"],
"exactOptionalPropertyTypes": false,
"declarationDir": "../../../types/playback/primitives"
},
"references": [
{ "path": "../../../../utils" },
{ "path": "../../core" },
{ "path": "../../network" },
{ "path": "../../media" }
],
"include": ["./*.ts", "./tests/**/*.ts"]
}