From b89f1e944cdc8ee63e52095e712ed3e8f0bab236 Mon Sep 17 00:00:00 2001 From: Christian Pillsbury Date: Wed, 17 Jun 2026 08:20:43 -0700 Subject: [PATCH] feat(spf): multi cdn failover (#1671) Co-authored-by: Claude Opus 4.8 (1M context) --- .../design/spf/features/multi-cdn-failover.md | 323 ++++++++++-------- .../spf/multi-cdn-failover-prior-art.md | 209 ++++++++++++ packages/spf/src/CLAUDE.md | 8 +- packages/spf/src/media/utils/cdn.ts | 38 ++- .../spf/src/media/utils/tests/cdn.test.ts | 21 +- packages/spf/src/network/fetch.ts | 17 + ...cdn-priority.ts => derive-cdn-priority.ts} | 17 +- .../playback/behaviors/dom/load-segments.ts | 2 +- .../behaviors/dom/setup-buffer-actors.ts | 24 +- .../src/playback/behaviors/dom/tsconfig.json | 1 + .../src/playback/behaviors/resolve-track.ts | 82 ++++- .../src/playback/behaviors/select-tracks.ts | 2 +- .../behaviors/setup-failover-monitor.ts | 113 ++++++ ...ty.test.ts => derive-cdn-priority.test.ts} | 22 +- .../behaviors/tests/resolve-track.test.ts | 1 + .../tests/setup-failover-monitor.test.ts | 83 +++++ .../behaviors/tests/track-switching.test.ts | 126 ++++++- .../src/playback/behaviors/track-switching.ts | 171 ++++++++-- .../spf/src/playback/behaviors/tsconfig.json | 1 + .../playback/engines/hls/engine-audio-only.ts | 33 +- .../spf/src/playback/engines/hls/engine.ts | 44 ++- .../playback/engines/hls/tests/engine.test.ts | 172 +++++++++- .../engines/hls/tests/failover-smoke.test.ts | 87 +++++ .../src/playback/primitives/failover-fetch.ts | 63 ++++ .../primitives/tests/failover-fetch.test.ts | 73 ++++ .../{behaviors => primitives}/track-types.ts | 0 .../spf/src/playback/primitives/tsconfig.json | 16 + packages/spf/vitest.config.ts | 6 +- tsconfig.json | 1 + 29 files changed, 1508 insertions(+), 248 deletions(-) create mode 100644 internal/design/spf/multi-cdn-failover-prior-art.md rename packages/spf/src/playback/behaviors/{resolve-cdn-priority.ts => derive-cdn-priority.ts} (86%) create mode 100644 packages/spf/src/playback/behaviors/setup-failover-monitor.ts rename packages/spf/src/playback/behaviors/tests/{resolve-cdn-priority.test.ts => derive-cdn-priority.test.ts} (86%) create mode 100644 packages/spf/src/playback/behaviors/tests/setup-failover-monitor.test.ts create mode 100644 packages/spf/src/playback/engines/hls/tests/failover-smoke.test.ts create mode 100644 packages/spf/src/playback/primitives/failover-fetch.ts create mode 100644 packages/spf/src/playback/primitives/tests/failover-fetch.test.ts rename packages/spf/src/playback/{behaviors => primitives}/track-types.ts (100%) create mode 100644 packages/spf/src/playback/primitives/tsconfig.json diff --git a/internal/design/spf/features/multi-cdn-failover.md b/internal/design/spf/features/multi-cdn-failover.md index c088b97a..6f09a15e 100644 --- a/internal/design/spf/features/multi-cdn-failover.md +++ b/internal/design/spf/features/multi-cdn-failover.md @@ -1,7 +1,7 @@ --- status: draft -date: 2026-06-05 -definition: sketched +date: 2026-06-08 +definition: implemented --- # Multi-CDN failover @@ -22,12 +22,16 @@ Two sub-features, mapping onto the two rule kinds in that model: so video / audio / text resolve from one host. This is the *active-pathway scope* the track-switching model lists for multi-cdn-failover. -2. **Failover** *(deferred)* — when requests to the active CDN fail too - often within a window, a **constraint** removes that CDN's tracks from - the candidate set during cooldown and the session behavior rotates the - active CDN. This is the *failed-CDN constraint* the model lists, and it - consumes [network-resilience](./network-resilience.md)'s per-host - circuit-breaker — its hard prerequisite. +2. **Failover** *(implemented)* — **site-adds, behavior-expires.** Fetch + sites add a CDN to a `failedCdns` set on a failed fetch (the *trip*); a + shared **constraint** (`excludeFailedCdns`) prunes that CDN's tracks from + the candidate set so the scope falls to the next CDN; `setupFailoverMonitor` + removes the CDN once a cooldown lapses (the *expiry*) and the scope returns + to it. This is the *failed-CDN constraint* the model lists. It shipped + **self-contained** — a cooldown timer, *not* + [network-resilience](./network-resilience.md)'s circuit-breaker. Retries are + a future refinement that would sit *below* the trip (so it sees post-retry + terminal failures), not a prerequisite. A **Media-src feature** for sources that genuinely require failover, with a **Player feature** surface at the failover tier (customer-customizable @@ -36,26 +40,30 @@ ambiguity reflects that split. ## Status -- **Composition:** sub-feature 1 (sticky CDN pick) is implemented in - `createSimpleHlsEngine` and `createHlsAudioOnlyEngine`. The - `resolveCdnPriority` behavior owns the `cdnPriority` signal (the - manifest-ordered CDN list, most-preferred first); the `preferActiveCdn` - scope rule (shared by the video + audio chains in `track-switching`) - narrows candidates to the highest-priority CDN with surviving tracks. - Failover (sub-feature 2) is not implemented — no constraints pass, no - per-CDN failure tracking, no rotation. -- **Definition depth:** sketched — sub-feature 1 has a populated - implementation surface + verification; sub-feature 2 stays at the - scope-and-constraints level pending its prerequisite. -- **Hard prerequisite (failover only):** - [network-resilience](./network-resilience.md). The failed-CDN - constraint consumes the foundation's per-host circuit-breaker / - retry-exhaustion state. Sub-feature 1 has no such dependency — it's - pure selection over the already-parsed candidate set. +- **Composition:** both sub-features are implemented in + `createSimpleHlsEngine` and the audio-only engine. `deriveCdnPriority` + owns the `cdnPriority` signal (manifest-ordered CDN list); the + `preferActiveCdn` scope rule narrows candidates to the highest-priority CDN + with surviving tracks (shared by the video + audio chains). For failover: + the `excludeFailedCdns` constraint prunes tracks whose CDN is in the + `failedCdns` set; fetch sites trip a CDN into `failedCdns` on a failed fetch + (`failoverFetch` for media playlists in `resolve-track`, `failoverFetchBytes` + for segments in `setup-buffer-actors`); `setupFailoverMonitor` owns `failedCdns` + and removes each CDN once its cooldown lapses. +- **Definition depth:** implemented. +- **Detection (self-contained, no hard prerequisite):** a CDN trips on the + **first terminal fetch failure** (network error or non-OK status); cooldown + is the only back-off. `setupFailoverMonitor` is a per-source cooldown timer, + *not* a circuit-breaker imported from + [network-resilience](./network-resilience.md). Retries are a future + refinement *below* the trip (so it would observe only post-retry terminal + failures), not a dependency — this feature ships without + `network-resilience` existing. - **Governing model:** [track-switching-model.md](../track-switching-model.md) - — multi-CDN is the canonical *constraint + scope* feature there. The - active-CDN scope is a soft filter in the rule chain; the failed-CDN - constraint is a hard filter in the (not-yet-built) constraints pre-pass. + — multi-CDN is the canonical *constraint + scope* feature there. Both halves + are now built: the active-CDN scope (soft filter in the rule chain) and the + failed-CDN constraint (hard filter in the constraints pre-pass — + `applyConstraints` + the `constraints` config slot now exist). ## How redundant streams are modeled @@ -65,7 +73,8 @@ redundant-streams manifest lists each rendition once per CDN (duplicate existing parser already emits one `Track` per entry with a unique id and its own absolute `url`. So the candidate set for each type *already* contains one variant per CDN. CDN identity is derived from each track -URL's origin (`getCdnId`); the set of CDNs is published as a single +URL's origin (`getCdnId`, overridable via engine config — e.g. to key on +Mux's `cdn=` query param); the set of CDNs is published as a single per-presentation ordered signal (`cdnPriority`, most-preferred first — mirroring HLS content steering's `PATHWAY-PRIORITY`). The *active* CDN is not stored: the scope derives it as the highest-priority `cdnPriority` @@ -74,38 +83,39 @@ CDN-tagged track id means `resolveTrack` / segment loading fetch from that CDN with no further plumbing. This list shape is what makes failover fall out cleanly: the failed-CDN -constraint (sub-feature 2) prunes a cooled-down CDN's tracks, so -"first-with-survivors" moves to the next CDN automatically and returns to -the primary when it recovers — no reactive rewrite of an "active" value. -Content steering, likewise, just reorders `cdnPriority` (pathway priority -as a sort key). +constraint prunes a cooled-down CDN's tracks, so "first-with-survivors" +moves to the next CDN automatically and returns to the primary when it +recovers — no reactive rewrite of an "active" value. Content steering, +likewise, just reorders `cdnPriority` (pathway priority as a sort key). -This is why sub-feature 1 needed **no parser change** and no new data -shape — only a selection behavior and a scope rule over existing tracks. +This is why the feature needed **no parser change** and no new data shape +— only selection behaviors, a scope rule, and a constraint over existing +tracks, plus a fetch decorator that records failures into `failedCdns`. ## Phases of complexity | Phase | Sub-feature | Kind | What | State | |---|---|---|---|---| -| Sticky CDN pick | 1 | scope | `resolveCdnPriority` publishes the manifest-ordered CDN list (`cdnPriority`); `preferActiveCdn` narrows every type's candidates to the highest-priority CDN with surviving tracks, falling through when nothing matches. Shared list → all types on one CDN | **Implemented** | -| Failed-CDN constraint | 2 | constraint | A constraint in the track-switching constraints pre-pass removes a cooled-down CDN's tracks from the candidate set. The scope then picks the next `cdnPriority` entry automatically. Requires the generic constraints phase (track-switching-model "Phase 2") to be built first | Deferred | -| Per-CDN failure tracking | 2 | — | Count per-CDN fetch failures within a window; mark a CDN unhealthy / in cooldown. Consumes `network-resilience`'s circuit-breaker | Deferred (prereq) | -| CDN priority override / steering | 2 | scope | Reorder `cdnPriority` to bias the pick (region-preferred, weighted, or content-steering's pathway priority). No reactive "active" rewrite needed — the order *is* the policy | Deferred | -| Customer CDN-id derivation | 2 | config | Pluggable `getCdnId` for non-origin identity. The origin-based default is built in; a config seam is anticipated | Deferred | +| Sticky CDN pick | 1 | scope | `deriveCdnPriority` publishes the manifest-ordered CDN list (`cdnPriority`); `preferActiveCdn` narrows every type's candidates to the highest-priority CDN with surviving tracks, falling through when nothing matches. Shared list → all types on one CDN | **Implemented** | +| Constraints pre-pass | 2 | — | `applyConstraints` + the `constraints` config slot in `setupTrackSwitching` (the hard-filter pre-pass that runs before the rule chain). Reusable by capability-probing | **Implemented** | +| Failed-CDN constraint | 2 | constraint | `excludeFailedCdns` prunes tracks whose CDN ∈ `failedCdns`; the scope falls to the next `cdnPriority` entry and snaps back on recovery | **Implemented** | +| Per-CDN failure tracking | 2 | — | **Site-adds, behavior-expires**: fetch sites trip a CDN into `failedCdns` on a failed fetch (`failoverFetch` / `failoverFetchBytes`); `setupFailoverMonitor` expires it after a cooldown. Self-contained (trip-on-first-failure + cooldown), not a `network-resilience` circuit-breaker | **Implemented** | +| Customer CDN-id derivation | 2 | config | Pluggable `getCdnId` (engine config) for non-origin identity, threaded to all four CDN-id sites; origin-based default | **Implemented** | +| CDN priority override / steering | 2 | scope | Reorder `cdnPriority` to bias the pick (region-preferred, weighted, or content-steering's pathway priority). No reactive "active" rewrite needed — the order *is* the policy | Deferred (content-steering) | ## What's in scope vs out of scope -**In scope:** -- Sticky per-presentation CDN selection (sub-feature 1, done) -- Failover via a track-switching constraint + active-CDN rotation - (sub-feature 2) -- Per-CDN health derived from `network-resilience`'s circuit-breaker -- Customer-configurable CDN-id derivation / rotation policy +**In scope (all implemented):** +- Sticky per-presentation CDN selection +- Failover via the failed-CDN constraint + derived active-CDN rotation +- Self-contained per-CDN failure tracking (trip-on-first-failure + cooldown) +- Customer-configurable CDN-id derivation (`getCdnId`) **Out of scope (separate cluster G sister features):** -- **[network-resilience](./network-resilience.md)** *(foundation, - prerequisite for failover)* — retry + backoff + circuit-breaker. - Multi-CDN consumes; doesn't reimplement. +- **[network-resilience](./network-resilience.md)** *(optional future + refinement, not a prerequisite)* — retry + backoff + error-classification. + Would sit *below* the failover trip to reduce false trips; failover ships + without it. - **[content-steering](./content-steering.md)** — HLS content-steering protocol (server-advertised, dynamically-updated host pool). Different mechanism than static redundant streams, but the *same* active-pathway @@ -129,7 +139,7 @@ shape — only a selection behavior and a scope rule over existing tracks. video and audio chains reference the *same* `preferActiveCdn` definition reading the *same* list, so they agree on the CDN even if their per-type track arrays differ. (The doc's earlier per-rendition lean is superseded.) -- **`cdnPriority` writer composition.** `resolveCdnPriority` is the sole +- **`cdnPriority` writer composition.** `deriveCdnPriority` is the sole writer today (publishes the manifest order). Failover needs no second writer — the failed-CDN constraint prunes tracks and the scope re-derives the active CDN. Content-steering would *reorder* `cdnPriority` (still a @@ -140,120 +150,153 @@ shape — only a selection behavior and a scope rule over existing tracks. the primary. This is why the array beats a single reactive `activeCdn` value — the failed-set information is applied once (in the constraint), not duplicated into an active-value rewrite. -- **Constraints phase is a shared prerequisite.** The failed-CDN - constraint can't land until the generic constraints pre-pass - (`applyConstraints` + the `constraints` config field + empty-playable-set - terminal state) is built into `setupTrackSwitching`. The seam exists - (`candidateSet` computed); the machinery does not. capability-probing - shares this prerequisite. +- **Constraints pre-pass (now built).** The failed-CDN constraint runs in + `setupTrackSwitching`'s `applyConstraints` pre-pass — the `constraints` + config slot, applied to `candidateSet` before the rule chain. + capability-probing can reuse it. One piece is deliberately *not* built: a + terminal "everything pruned" state — today an all-CDNs-failed candidate set + is empty and the prior pick is left in place (see *Follow-up candidates*). - **Live + multi-CDN.** During live playback the reload loop re-resolves - the presentation; `resolveCdnPriority` re-publishes only when the CDN set + the presentation; `deriveCdnPriority` re-publishes only when the CDN set changes (idempotent for a stable manifest). Cross-feature with [live-stream-support](./live-stream-support.md) (not yet implemented). -- **Priority state across source changes.** `cdnPriority` tears down with - the source via the resolved/unresolved cascade (per - [source-replacement](./source-replacement.md)); per-host circuit-breaker - state in `network-resilience` may outlive a source. +- **Failover state is per-source.** Both `cdnPriority` and `failedCdns` tear + down with the source via the resolved/unresolved cascade (per + [source-replacement](./source-replacement.md)) — `setupFailoverMonitor` clears + `failedCdns` and its cooldown timers on unload, so no failover state leaks + across sources. (If `network-resilience` lands later, its per-host state may + choose to outlive a source — that's its call, not failover's.) ## Open questions -- **Constraints-phase shape.** How `applyConstraints` and the - empty-playable-set terminal state are modeled — owned by the - track-switching constraints work, consumed here. (See - [track-switching-model.md](../track-switching-model.md) → *Fitting the - model to the track-switching behavior*.) -- **Failure-window policy.** Threshold count / window length / cooldown - duration for marking a CDN unhealthy. Empirical; lives with - `network-resilience`'s circuit-breaker. -- **CDN-id derivation configurability.** Origin-based `getCdnId` is the - default; whether/where to expose a consumer override (config field vs. - rule config view) is deferred until a non-origin case appears. -- **Composition with content-steering.** Static redundant CDNs + - dynamic steered host pool: does steering replace, intersect, or - reprioritize the candidate CDNs? Open until content-steering lands; the - reorderable `cdnPriority` list keeps it tractable (steering reorders it). +- **Composition with content-steering.** Static redundant CDNs + a dynamic + steered host pool: does steering replace, intersect, or reprioritize the + candidate CDNs? Open until content-steering lands; the reorderable + `cdnPriority` list keeps it tractable (steering reorders it). -### Resolved during sub-feature 1 implementation +## Follow-up candidates -- **Signal shape** → a per-presentation ordered `cdnPriority` list (active - = first-with-survivors, derived), not a single stored `activeCdn` value - and not per-rendition. The list makes failover a pure constraint and - composes with content-steering as a reorder. Matches the cross-type - coherence requirement (one shared list). -- **CDN identity** → URL origin (`getCdnId`); configurable derivation - deferred. -- **Manifest syntax / parser** → no change. Redundant variants already - parse as separate per-CDN tracks; no `alternateUris` field needed. +Known, intentionally-deferred refinements — none block the feature; worth +tracking as a future effort: + +- **No cooldown extension on re-failure.** A CDN's removal is scheduled when it + first enters `failedCdns`; re-failing it mid-cooldown doesn't push the deadline + out (set-membership watch). Fine for trip-on-first; revisit with a + windowed/decaying health metric. +- **Flapping.** A flaky-but-not-dead CDN can oscillate — trip → cooldown lapses → + re-preferred (it's `cdnPriority[0]`) → fails again. No hysteresis or + growing back-off on repeated trips. +- **All-CDNs-down has no terminal state.** When every CDN is pruned the candidate + set is empty and the prior pick is silently left in place; a distinct "nothing + playable" state is unmodeled (shared with the constraints-pre-pass work). +- **HTTP-status classification is coarse.** The trip fires on a thrown fetch or a + non-OK media-playlist status; finer classification (5xx-with-body vs 4xx, + segment-side status codes) is deferred to `network-resilience`. +- **Retries below the trip.** `network-resilience` retry/backoff would sit under + the fetch sites so the trip sees only post-retry terminal failures (fewer false + trips). A refinement, not a dependency. +- **`switchAudioTrack` config tidiness (cosmetic).** Audio spreads `...config`, so + video-only ABR fields ride into the shared ranker harmlessly (audio has no + `bandwidthState`). A cross-cutting-only shared config type would keep them out. + +### Resolved during implementation + +- **Signal shape** → a per-presentation ordered `cdnPriority` list (active = + first-with-survivors, derived), not a stored `activeCdn` and not per-rendition. + The list makes failover a pure constraint and composes with content-steering as + a reorder; matches the one-shared-list cross-type coherence requirement. +- **Failover detection** → **site-adds, behavior-expires**: fetch sites trip on + the first terminal failure; `setupFailoverMonitor` expires after a cooldown. + Self-contained — no `network-resilience` circuit-breaker dependency. +- **CDN-id derivation** → configurable via the `getCdnId` engine config (origin + default), threaded to all four CDN-id sites so the keys stay comparable. +- **Constraints pre-pass** → built (`applyConstraints` + `constraints` config + slot in `setupTrackSwitching`). +- **Parse failures don't trip** — a 200 with an unparseable body is a content + issue, not CDN unavailability; only fetch/non-OK failures trip. +- **Manifest syntax / parser** → no change; redundant variants already parse as + separate per-CDN tracks (no `alternateUris` field). - **Architecture** → constraint + scope in the track-switching model, not active-URI rotation in `resolveTrack`. ## Implementation surface -- **`packages/spf/src/media/utils/cdn.ts`** — `getCdnId(url)` (origin-based - CDN identity) and `getOrderedCdnIds(presentation)` (distinct CDNs in - manifest order; head = primary). -- **`packages/spf/src/playback/behaviors/resolve-cdn-priority.ts`** — - `resolveCdnPriority` behavior + `ResolveCdnPriorityState`. Machine reactor - on `presentation-unresolved` ↔ `presentation-resolved`; owns the - `cdnPriority` signal; publishes `getOrderedCdnIds(presentation)` (skipping - the write when the CDN set is unchanged), clears on exit. -- **`packages/spf/src/playback/behaviors/track-switching.ts`** — - `preferActiveCdn` scope rule (soft filter on `cdnPriority`: narrow to the - highest-priority CDN with surviving tracks), added to both variants' - chains: `[filterByUserSelection, preferActiveCdn, rankByBandwidth]`. - `SwitchableTrack` gains `url` (the rule's input). -- **`packages/spf/src/playback/engines/hls/engine.ts` + - `engine-audio-only.ts`** — `resolveCdnPriority` composed after - `resolvePresentation`; `cdnPriority?: string[]` added to both engine state - interfaces. +- **`media/utils/cdn.ts`** — `getCdnId(url)` (origin-based default) + the + `GetCdnId` type; `getOrderedCdnIds(presentation, getCdnId?)`; + `addFailedCdn(failed, cdn)` (pure, idempotent dedup-append). +- **`playback/behaviors/derive-cdn-priority.ts`** — `deriveCdnPriority` owns + `cdnPriority` (publishes `getOrderedCdnIds` on resolve, skips unchanged + writes, clears on exit). +- **`playback/behaviors/setup-failover-monitor.ts`** — `setupFailoverMonitor` + owns `failedCdns`; per-source, watches the set and schedules a cooldown + removal per CDN, clears on src unload. Config `failover?: { cooldownMs }`. +- **`playback/behaviors/resolve-track.ts`** — `failoverFetch(state, config)` + decorates the media-playlist fetch (`fetchResolvableText`); a failed/non-OK + fetch adds the CDN to `failedCdns` via `addFailedCdn`. +- **`playback/behaviors/dom/setup-buffer-actors.ts`** — `failoverFetchBytes` + decorates the per-type segment fetch (`trackedFetch` / `fetchStream`) the same way. +- **`playback/behaviors/track-switching.ts`** — `preferActiveCdn` scope + + `excludeFailedCdns` constraint (shared by the video + audio chains via the + `CdnRuleConfig` view that carries `getCdnId`); `applyConstraints` pre-pass + + `constraints` config slot; `SwitchableTrack` gains `url`. +- **`playback/engines/hls/engine.ts` + `engine-audio-only.ts`** — + `deriveCdnPriority` + `setupFailoverMonitor` composed after + `resolvePresentation`; `failover?` + `getCdnId?` engine config; `cdnPriority?` + + `failedCdns?` engine state. +- **`network/fetch.ts`** — `FetchText` type + `fetchResolvableText` default + (fetch → reject on non-OK → text), the text analog of `FetchBytes`. -State signal: `cdnPriority?: string[]` (CDN origins, most-preferred first), -owned by `resolveCdnPriority`, read optionally by `preferActiveCdn`. +State signals: `cdnPriority?: string[]` (owned by `deriveCdnPriority`) and +`failedCdns?: string[]` (owned by `setupFailoverMonitor`; tripped by the fetch +sites, read by the `excludeFailedCdns` constraint). ## Verification -Sub-feature 1: +- `media/utils/tests/cdn.test.ts` — `getCdnId` (origin; same/different host; + scheme+port; unparseable fallback); `getOrderedCdnIds` (order; dedupe; single; + unresolved → `[]`); `addFailedCdn` (append; order; idempotent same-reference). +- `playback/behaviors/tests/derive-cdn-priority.test.ts` — publishes the + manifest-ordered list; single-CDN; skips the write on a same-CDN swap; updates + on reorder; clears on unload/destroy; re-publishes after reset. +- `playback/behaviors/tests/setup-failover-monitor.test.ts` — a tripped CDN is + removed once its cooldown lapses; independent per-CDN cooldowns; clears + `failedCdns` on src unload; sensible cooldown default. +- `playback/behaviors/tests/track-switching.test.ts` — `preferActiveCdn` (narrow + to highest-priority surviving CDN; fall-through; cross-type coherence); + `excludeFailedCdns` + `applyConstraints` (prune failed CDNs; order-independence; + failover via the constraint). +- `playback/engines/hls/tests/engine.test.ts` — integration: redundant-stream → + `cdnPriority` = manifest order + pick on primary; reordering re-narrows; + **auto-failover** (a failing media-playlist fetch trips the CDN and selection + falls over to the backup, no external write); a **custom `getCdnId`** (keyed on + a query param) honored across `cdnPriority`, the trip, and the constraint/scope + end-to-end. +- `playback/engines/hls/tests/failover-smoke.test.ts` — **gated live smoke test** + (behind `VITE_FAILOVER_SMOKE`) against a real Mux `?redundant_streams=true` + source: block one origin → trip → failover → recovery. Skipped by default. -- `media/utils/tests/cdn.test.ts` — `getCdnId` (origin extraction; - same/different host; scheme+port; unparseable fallback); - `getOrderedCdnIds` (distinct CDNs in order; dedupe; single-CDN; - unresolved → `[]`). -- `playback/behaviors/tests/resolve-cdn-priority.test.ts` — publishes the - manifest-ordered CDN list; single-CDN source; skips the write when a - resolved swap keeps the same CDNs; updates when the order changes; clears - on src unload + on destroy; re-publishes after a src reset. -- `playback/behaviors/tests/track-switching.test.ts` (`preferActiveCdn` - block) — narrows to the highest-priority CDN overriding manifest track - order; keeps the pick on the primary; falls through to the next CDN when - the first has no survivors; falls through to all when none match; no-op - when `cdnPriority` absent; re-picks reactively when the order changes - (steering/failover seam); same scope applied to the audio chain - (cross-type coherence). -- `playback/engines/hls/tests/engine.test.ts` — integration: a - redundant-stream presentation yields `cdnPriority` = manifest order and a - video selection on the primary; reordering `cdnPriority` re-narrows and - the selection follows. +Deferred: -Out of scope / deferred: - -- End-to-end + sandbox verification against a real Mux - `?redundant_streams=true` source (needs a fixture). -- All of sub-feature 2 (failover): blocked on the constraints phase + - `network-resilience`. +- E2e through the html player + real MSE — deferred; apps/e2e pages are generated + from `media.ts`, so a redundant source would sweep into every generic + visual + spec. The engine smoke test covers the round-trip (including recovery, which + isn't observable at the player DOM). +- A dedicated segment-failover integration test — the trip logic is unit-covered + (`addFailedCdn`) and mirrors the tested media-playlist path. ## Related features - **[track-switching-model.md](../track-switching-model.md)** *(governing model)* — multi-CDN is its canonical constraint + scope feature; the active-CDN scope is implemented against the rule chain it specifies. -- **[network-resilience](./network-resilience.md)** *(hard prerequisite - for failover)* — per-host circuit-breaker the failed-CDN constraint - consumes. +- **[network-resilience](./network-resilience.md)** *(optional refinement, + not a prerequisite)* — retry/backoff below the trip would reduce false + trips; failover ships self-contained. - **[content-steering](./content-steering.md)** — dynamic host-pool sibling; shares the active-pathway scope shape (`cdnPriority` as a reorderable reflected list — pathway priority as a sort key). -- **[capability-probing](./capability-probing.md)** — shares the - not-yet-built constraints pre-pass with the failed-CDN constraint. +- **[capability-probing](./capability-probing.md)** — can reuse the + constraints pre-pass (now built for the failed-CDN constraint). - **[live-stream-support](./live-stream-support.md)** *(not implemented)* — reload-loop re-resolution; `cdnPriority` is republished only when the CDN set changes. @@ -264,6 +307,10 @@ Out of scope / deferred: ## See also +- [multi-cdn-failover-prior-art.md](../multi-cdn-failover-prior-art.md) — + how eight OSS players model CDN redundancy/failover, read against this + design (two architectural families, content-steering convergence, and the + prior art behind the 300s cooldown default + the open follow-ups) - [track-switching-model.md](../track-switching-model.md) — the rule model (constraints → soft filters → ranker) this feature composes into - [clusters.md § Selection resilience](./clusters.md#selection-resilience) @@ -271,8 +318,8 @@ Out of scope / deferred: `network-resilience`'s response-error handling - [clusters.md § Selection / filtering across clusters](./clusters.md#selection--filtering-across-clusters) — cluster G's role: alternate-CDN selection within the chosen track -- [network-resilience.md](./network-resilience.md) — circuit-breaker - foundation the failover tier consumes +- [network-resilience.md](./network-resilience.md) — retry/backoff that + would sit below the failover trip (optional refinement, not consumed today) - [SPF Epics Working Doc](https://www.notion.so/35f97a7f89d08123a13fecab1ca1cac4) — source material; epic #9 (Multi-CDN Failover) - [Mux Video — `?redundant_streams=true`](https://www.mux.com/docs/guides/play-back-on-multiple-cdns) diff --git a/internal/design/spf/multi-cdn-failover-prior-art.md b/internal/design/spf/multi-cdn-failover-prior-art.md new file mode 100644 index 00000000..d6b9e584 --- /dev/null +++ b/internal/design/spf/multi-cdn-failover-prior-art.md @@ -0,0 +1,209 @@ +--- +status: draft +date: 2026-06-12 +--- + +# Multi-CDN failover: prior art + +> A survey of how eight open-source players and playback engines model CDN +> redundancy and failover, read against SPF's design. It backs +> [features/multi-cdn-failover.md](./features/multi-cdn-failover.md), +> [features/content-steering.md](./features/content-steering.md), and +> [features/network-resilience.md](./features/network-resilience.md), and +> informs the constraint+scope framing in +> [track-switching-model.md](./track-switching-model.md). Iterate freely — +> this is a reference frame, not a status record. + +## Scope + +The research question: when a source publishes the same content on more than +one CDN (e.g. Mux Video's `?redundant_streams=true`), how do existing players +(a) represent that redundancy, (b) decide a CDN has failed, (c) recover, and +(d) compose with HLS/DASH Content Steering? + +Surveyed: video.js v8 http-streaming (VHS), hls.js, dash.js, shaka-player, +rx-player, Media3/ExoPlayer, VLC, OSMF. Citations are against each project's +repository (see *Sources*) at survey time and may drift — verify before +quoting as current. + +## The two architectural families + +Every player that does real failover falls into one of two camps for **how +redundancy is modeled**. This is the axis SPF is most interesting against. + +**Family A — URL/URI rotation at the resource level.** The segment (or +BaseURL) carries *multiple URIs*; failover picks a different URI for that +resource. Redundancy lives *inside* the resource. + +- [**shaka**](https://github.com/shaka-project/shaka-player) — segments expose + `getUris()` → an array; `NetworkingEngine` rotates via + `index = request.attempt % request.uris.length` + (`lib/net/networking_engine.js`). +- [**dash.js**](https://github.com/Dash-Industry-Forum/dash.js) — multiple + `` per MPD node; `BaseURLController` selects per node + (`src/streaming/controllers/BaseURLController.js`). +- [**rx-player**](https://github.com/canalplus/rx-player) — `ICdnMetadata[]` + per Representation; a `CdnPrioritizer` picks among them + (`src/core/fetchers/cdn_prioritizer.ts`). +- [**media3 (DASH path)**](https://github.com/androidx/media) — + `BaseUrlExclusionList` filters `BaseUrl` objects + (`libraries/exoplayer_dash/.../BaseUrlExclusionList.java`). +- [**VLC**](https://github.com/videolan/vlc) — parses multiple `` but + *only ever uses `baseUrls.front()`* + (`modules/demux/adaptive/playlist/BasePlaylist.cpp`) — decorative. + +**Family B — pathway/ladder selection.** Redundant variants are *separate +playlists/levels* grouped by a pathway id; failover switches which whole +ladder is active. Redundancy lives *above* the resource. + +- [**hls.js**](https://github.com/video-dev/hls.js) — redundant streams become + separate `Level`s; pathway ids auto-assigned `"."`, `".."`, `"..."`; the + `ContentSteeringController` swaps ladders + (`src/controller/level-controller.ts`, + `src/controller/content-steering-controller.ts`). +- [**VHS**](https://github.com/videojs/http-streaming) — separate playlists + keyed by `PATHWAY-ID || serviceLocation`; exclusion + steering pathway switch + (`src/playlist-controller.js`). +- **media3 (HLS path)** — `HlsRedundantGroup` maps pathway ids → playlist URLs + (`libraries/exoplayer_hls/.../playlist/HlsRedundantGroup.java`). + +### Where SPF sits + +Family B *in spirit* — redundant variants parse as separate candidate tracks, +failover is a ladder-level concept — but expressed **declaratively as a +selection constraint in the track-switching rule chain** rather than +imperative "switch pathway" controller logic. None of the eight do this. The +closest mental model is hls.js / media3 "pathway = a complete ladder," but +they all *imperatively* reassign level/playlist indices on switch +(`reassignFragmentLevelIndexes()`, `switchPathway()`). SPF makes failover fall +out of the same `candidateSet` computed that already does ABR/track selection: +"prune the failed CDN's tracks, scope falls to the next entry" is structurally +the same operation as any other filter rule. That is the genuinely novel +framing, and it is why the feature needed no parser change and no dedicated +failover state machine — see +[features/multi-cdn-failover.md § How redundant streams are modeled](./features/multi-cdn-failover.md). + +## Comparison by axis + +| Player | Model | Trip trigger | Recovery | CDN identity | Scope | +|---|---|---|---|---|---| +| **SPF (v10)** | Separate candidate tracks; failover = selection constraint | **First terminal fetch failure** (playlist or segment), no threshold | 300s cooldown expiry (config) | **URL origin, configurable `getCdnId`** | Per-presentation shared list | +| **VHS** | Separate playlists by pathway | First failure → temporal exclude; error *count* → permanent (`maxPlaylistRetries`) | Temporal expiry; permanent past threshold; last-rendition fallback clears others | `PATHWAY-ID \|\| serviceLocation` | Per-playlist + steering | +| **hls.js** | Separate levels by pathway | Cumulative **error threshold/retries**, then penalty box | **300s** penalty cooldown | `PATHWAY-ID` (auto-dotted) | Per-level + per-pathway | +| **dash.js** | Multiple BaseURLs per node | **First failure** → blacklist serviceLocation | Blacklist expiry; **indefinite by default** unless steering TTL | `serviceLocation` + DVB priority/weight | Per-node, sticky-cached | +| **shaka** | Multiple URIs per segment | Per-request retry rotation (modulo) | **Stateless** — each segment restarts at `uris[0]`; steering ban = 60s | serviceLocation / pathway | Per-request | +| **rx-player** | `ICdnMetadata[]` per Representation | First failure → downgrade + per-CDN retry counter → permanent at maxRetry | ~60s downgrade¹; `priorityChange` event pivots mid-backoff | `id` (≈ serviceLocation), baseUrl fallback | Per-segment, global prioritizer | +| **media3** | DASH: BaseUrls; HLS: redundant groups | **First failure, selective HTTP codes** (403/404/410/416/500/503) | **Asymmetric: 300s location / 60s track** | serviceLocation / pathway | 3 layers: chunk / track / location | +| **VLC** | Parses BaseURLs, uses only first | **No inter-CDN failover** — HTTP 3xx redirects only (max 3) | None | hostname/port (implicit) | Per-chunk redirect | +| **OSMF** | `serverBaseURLs[]` parsed but unused | **None** — retries same URL on timeout | None | manifest baseURL | Per-fragment | + +¹ rx-player's `DEFAULT_CDN_DOWNGRADE_TIME` reads `60` ms in `default_config.ts` +— most likely a typo for seconds; its tests use `5000`. Treat the intent as +"seconds," the value as unverified. + +## Content Steering: near-universal convergence + +The modern players implement it almost identically, which validates SPF's +choice to name its ordered list `cdnPriority` after HLS Content Steering's +`PATHWAY-PRIORITY`: + +- **hls.js, VHS, dash.js, shaka, media3** all parse a steering manifest with + `PATHWAY-PRIORITY` / `SERVICE-LOCATION-PRIORITY`, `PATHWAY-CLONES` (HOST / + PARAMS URI rewriting), TTL-based reload, and `_HLS_pathway` / `_HLS_throughput` + (or `_DASH_*`) query hints. Steering **reorders** the priority list and can + **synthesize** new CDN pathways via clones. +- **rx-player** built `CdnPrioritizer` *specifically* as the steering + substrate but hasn't wired steering in — "waiting for the spec to be + standardized and relied on in the wild" (`cdn_prioritizer.ts` class comment). +- **VLC, OSMF** — absent. + +The industry consensus — "steering reorders the priority list and clone-rewrites +URIs" — is exactly the design [features/multi-cdn-failover.md](./features/multi-cdn-failover.md) +anticipates: `cdnPriority` is a reorderable list a steering behavior would +write (pathway priority as a sort key). The one piece everyone else has that +SPF would add later: **pathway clones** (HOST/PARAMS rewriting to synthesize +CDNs absent from the manifest). SPF's configurable `getCdnId` is the seam where +clone-rewritten URLs would need consistent identity. Tracked in +[features/content-steering.md](./features/content-steering.md). + +## Ideas worth adopting + +Mapped to the *Follow-up candidates* in +[features/multi-cdn-failover.md](./features/multi-cdn-failover.md): + +1. **Asymmetric cooldowns (media3, hls.js).** media3 uses **300s for + location/CDN exclusion vs 60s for track exclusion** — the principle being + "CDN failures are infrastructure problems (longer) than bitrate issues + (shorter)." This is the prior art behind bumping SPF's default cooldown + from 30s to **300s**. The next refinement — cooldown-extension on re-failure + — is what hls.js's penalty box and media3's exclusion already do. + +2. **rx-player's `priorityChange` event** is the most elegant recovery design: + when a downgraded CDN's cooldown expires it *interrupts an in-flight backoff + wait* and pivots immediately. Push-driven recovery, not "next request happens + to re-check." Relevant if SPF ever wants recovery faster than the next + natural fetch. + +3. **media3's `LoadErrorHandlingPolicy` / `FallbackSelection` / + `FallbackOptions`** is the cleanest *abstraction*: a pluggable policy that, + given `FallbackOptions(numberOfLocations, numberOfTracks)`, returns either a + location-fallback or a track-fallback with a duration — and **prefers + location fallback over track fallback** when both are available. It cleanly + separates "this is a CDN problem" from "this is a quality problem." SPF's + `applyConstraints` pre-pass is the analogous seam; a mature policy layer on + top would look like this. + +4. **Selective HTTP-status classification (media3; VHS's 410→permanent, + 429→Retry-After).** Directly addresses the *HTTP-status classification is + coarse* follow-up. media3 is the reference for which codes should trip vs + retry. + +5. **VHS's last-rendition fallback** — when excluding the final playlist, + proactively clear other temporal exclusions instead of erroring — is exactly + the *all-CDNs-down has no terminal state* gap. Their answer: don't have a + terminal state; re-admit everyone and retry rather than hard-fail. + +6. **shaka's stateless per-request rotation** is the opposite end from SPF — no + cross-request memory, every segment starts at `uris[0]`. Dead simple, no + flapping logic, but no stickiness either. Worth knowing as the minimal + baseline; SPF's sticky `cdnPriority` + cooldown is deliberately more stateful. + +## Where SPF is distinctive + +- **CDN identity from URL origin, not a manifest signal.** Everyone else keys + on an explicit manifest token (`serviceLocation` for DASH, `PATHWAY-ID` for + HLS). SPF derives identity from the URL because Mux's `?redundant_streams=true` + doesn't emit pathway tags — and `getCdnId` makes it configurable (origin vs + `cdn=` param). A gap-filler the spec-bound players don't need but also can't + do; the right call for untagged redundancy. +- **Failover as a pure rule in an existing selection chain**, not a dedicated + controller with its own state. Every other player carries a distinct object + (`ContentSteeringController`, `BaseURLSelector`, `CdnPrioritizer`, + `BaseUrlExclusionList`). SPF's failover is a constraint over tracks plus a + per-source cooldown timer (`setupFailoverMonitor`) — a smaller surface than + anyone else. +- **Trip-on-first-failure with cooldown-only backoff** matches dash.js and the + spirit of media3, but is more aggressive than hls.js / rx-player, which both + count errors before tripping. SPF's bet — "absorbing transient blips is the + retry layer's job, once it exists" — is reasonable but means it is more + sensitive to a single blip than hls.js until network-resilience lands. + +## Sources + +Surveyed repositories (paths are within each repo, read against its default +branch at survey time — verify before quoting as current): + +- video.js v8 http-streaming (VHS) — https://github.com/videojs/http-streaming — `src/` +- hls.js — https://github.com/video-dev/hls.js — `src/controller/` +- dash.js — https://github.com/Dash-Industry-Forum/dash.js — `src/streaming/`, `src/dash/controllers/` +- shaka-player — https://github.com/shaka-project/shaka-player — `lib/net/`, `lib/util/content_steering_manager.js` +- rx-player — https://github.com/canalplus/rx-player — `src/core/fetchers/` +- Media3 / ExoPlayer — https://github.com/androidx/media — `libraries/exoplayer*/` +- VLC — https://github.com/videolan/vlc — `modules/demux/adaptive/` +- OSMF — https://github.com/denivip/OSMF — `framework/OSMF/org/osmf/net/httpstreaming/` + +External references: + +- [Mux Video — `?redundant_streams=true`](https://www.mux.com/docs/guides/play-back-on-multiple-cdns) +- [HLS Content Steering (draft-pantos-hls-rfc8216bis)](https://datatracker.ietf.org/doc/html/draft-pantos-hls-rfc8216bis) +- [DASH-IF Content Steering](https://dashif.org/docs/DASH-IF-CTS-00XX-Content-Steering-Community-Review.pdf) diff --git a/packages/spf/src/CLAUDE.md b/packages/spf/src/CLAUDE.md index bb57dddc..8aa4331d 100644 --- a/packages/spf/src/CLAUDE.md +++ b/packages/spf/src/CLAUDE.md @@ -10,6 +10,7 @@ src/ media/ CML-like media building blocks — types, parsers, ABR, buffer logic, MSE/VTT primitives network/ HTTP fetch utilities, chunked-stream iterables playback/ Playback domain — composes core+media+network into engines + primitives/ Signal-aware playback building blocks composed by behaviors (no behavior lifecycle) behaviors/ Compositional behaviors that drive playback (use signals/effects) actors/ Actor factories used inside behaviors engines/ Assembled playback engines (currently just hls/) @@ -31,8 +32,10 @@ src/ | `core/` → DOM | ❌ | `core/tsconfig.json` `lib` excludes `DOM` | | `media/` (non-dom) → DOM | ❌ | `media/tsconfig.json` `lib` excludes `DOM` | | `network/` → DOM | ❌ | `network/tsconfig.json` `lib` excludes `DOM` | +| `playback/primitives/` → DOM | ❌ | tsconfig `lib` excludes `DOM` | | `playback/behaviors/` (non-dom) → DOM | ❌ | tsconfig `lib` excludes `DOM` | | `playback/actors/` (non-dom) → DOM | ❌ | tsconfig `lib` excludes `DOM` | +| `playback/behaviors/` → `playback/primitives/` | ✅ | references | | `playback/` → `core/`, `network/`, `media/` | ✅ | references in playback tsconfigs | | `playback/engines/hls/` → `playback/behaviors/`, `playback/actors/` | ✅ | references | | `playback/engines/hls/` → `core/`, `media/` | ✅ — engines compose primitives directly | references | @@ -42,12 +45,13 @@ The substance: `core/`, `media/`, `network/` are framework-agnostic foundations. ## Where to put new code - **Pure media/streaming logic** (parsers, types, selection algorithms, MSE/VTT helpers without signals): `media/` or `media/dom/`. Must not import from `core/`. +- **Signal-aware playback primitive** (a composable building block that reads/writes `core` signals but has no behavior lifecycle — no `effect`/`computed`/`subscribe`, not invoked by composition): `playback/primitives/`. These are composed *by* behaviors/actors/engines (e.g. `failoverFetch`, a fetch decorator that reads selected-track signals at fetch time; `track-types`, the per-type config bundles). They live at the `playback/` layer rather than `media/`/`network/` precisely because they touch `core/` — that's the line below. - **Compositional behavior driving state** (uses `effect`/`computed`/`update` against owners or state signals): `playback/behaviors/` or `playback/behaviors/dom/`. - **Actor factories** (long-lived stateful units that receive messages): `playback/actors/` or `playback/actors/dom/`. - **Engine compositions** (wiring behaviors+actors+config into a `createComposition` call): `playback/engines//`. - **Generic, framework-agnostic utilities** that aren't media-specific: prefer `@videojs/utils` over creating new homes inside spf. -If a module looks like a primitive but reaches into `core/`, that's a smell — consider whether the signal binding can move to the call site (see `onMediaSourceReadyStateChange` for a callback-shaped primitive that lets the consumer create the signal). +If a module looks like a `media/`/`network/` primitive but reaches into `core/`, that's a smell — first consider whether the signal binding can move to the call site (see `onMediaSourceReadyStateChange` for a callback-shaped primitive that lets the consumer create the signal), keeping the primitive itself `core`-free. Only when a building block genuinely needs *live* signal access — e.g. `failoverFetch` is constructed once but its returned fetch reads `presentation`/selected-track signals lazily on every call — does it belong at the `playback/` layer, in `playback/primitives/`. Conversely: if a function inside `playback/behaviors/` (or `playback/actors/`) has no `core/` dependency — no signals, effects, or reactors — it probably belongs in a layer below (`media/`, `network/`, or `@videojs/utils`). Same layering principle, opposite direction. When reviewing a behavior or actor file, scan its top-level helpers; any pure data-manipulation / lookup / format-handling code with no reactive concerns is a candidate to extract. @@ -71,7 +75,7 @@ Internal paths are not part of the public API. Don't import from `@videojs/spf/p ## Vitest projects `packages/spf/vitest.config.ts` shards tests by area: -- `core`, `media`, `network`, `behaviors` — Node, no browser +- `core`, `media`, `network`, `behaviors` — Node, no browser (the `behaviors` project also covers `playback/actors/` and `playback/primitives/`) - `dom` — Chromium via Playwright, covers all `**/dom/**/*.test.ts` across subtrees - `playback-engines` — Chromium, covers engines/ - `types` — type-only tests via tsgo diff --git a/packages/spf/src/media/utils/cdn.ts b/packages/spf/src/media/utils/cdn.ts index 97020e7a..7cb9515f 100644 --- a/packages/spf/src/media/utils/cdn.ts +++ b/packages/spf/src/media/utils/cdn.ts @@ -1,13 +1,19 @@ import type { MaybeResolvedPresentation, TrackType } from '../types'; /** - * Identify the CDN a URL is served from, used to group redundant-stream - * variants that point at the same content on different hosts. Defaults to the - * URL's origin (scheme + host + port); falls back to the raw string when the - * URL can't be parsed, so the return value is always a stable grouping key. - * - * Sub-feature 1 (sticky CDN pick) uses origin-based identity; a more advanced - * or consumer-configurable derivation can replace this default later. + * Derive a stable grouping key for the CDN a URL is served from. Synchronous and + * pure (deliberately not a `resolve*` — no fetch). Consumers override the + * default via the engine's `getCdnId` config (e.g. to key on Mux's `cdn=` query + * param instead of the host); every CDN-identity site reads that same function + * so keys stay comparable across `cdnPriority`, `failedCdns`, and the + * track-switching constraint + scope. + */ +export type GetCdnId = (url: string) => string; + +/** + * Default {@link GetCdnId}: the URL's origin (scheme + host + port); falls back + * to the raw string when the URL can't be parsed, so the return value is always + * a stable grouping key. */ export function getCdnId(url: string): string { try { @@ -34,9 +40,11 @@ const CDN_TYPE_PRIORITY: Record = { video: 0, audio: 1, text: * * Redundant-stream sources list the same content on multiple hosts (e.g. Mux's * `?redundant_streams=true`), so each host contributes its own candidate tracks; - * this collapses them to the set of CDNs across every track type. + * this collapses them to the set of CDNs across every track type. The CDN-id + * derivation defaults to {@link getCdnId}; pass a consumer-configured `getId` to + * key on something other than origin. */ -export function getOrderedCdnIds(presentation: MaybeResolvedPresentation): string[] { +export function getOrderedCdnIds(presentation: MaybeResolvedPresentation, getId: GetCdnId = getCdnId): string[] { const seen = new Set(); const ids: string[] = []; // Stable sort keeps manifest order among same-type selection sets. @@ -46,7 +54,7 @@ export function getOrderedCdnIds(presentation: MaybeResolvedPresentation): strin for (const selectionSet of selectionSets) { for (const switchingSet of selectionSet.switchingSets) { for (const track of switchingSet.tracks) { - const id = getCdnId(track.url); + const id = getId(track.url); if (seen.has(id)) continue; seen.add(id); ids.push(id); @@ -55,3 +63,13 @@ export function getOrderedCdnIds(presentation: MaybeResolvedPresentation): strin } return ids; } + +/** + * Add a CDN id to a failed-CDN list, preserving order and ignoring duplicates. + * Idempotent: re-adding an already-present id returns the same array reference + * (so a no-op trip doesn't churn the `failedCdns` signal). The failover trip in + * `resolve-track` and the segment loaders feed this into `failedCdns` via `update`. + */ +export function addFailedCdn(failed: string[] | undefined, cdn: string): string[] { + return failed?.includes(cdn) ? failed : [...(failed ?? []), cdn]; +} diff --git a/packages/spf/src/media/utils/tests/cdn.test.ts b/packages/spf/src/media/utils/tests/cdn.test.ts index 8ef38355..86a6e852 100644 --- a/packages/spf/src/media/utils/tests/cdn.test.ts +++ b/packages/spf/src/media/utils/tests/cdn.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { MaybeResolvedPresentation } from '../../types'; -import { getCdnId, getOrderedCdnIds } from '../cdn'; +import { addFailedCdn, getCdnId, getOrderedCdnIds } from '../cdn'; const presentationWith = (urlsByType: { video?: string[]; @@ -119,3 +119,22 @@ describe('getOrderedCdnIds', () => { expect(getOrderedCdnIds(presentation)).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']); }); }); + +describe('addFailedCdn', () => { + it('appends to an undefined or empty list', () => { + expect(addFailedCdn(undefined, 'https://cdn-a.example.com')).toEqual(['https://cdn-a.example.com']); + expect(addFailedCdn([], 'https://cdn-a.example.com')).toEqual(['https://cdn-a.example.com']); + }); + + it('appends in order', () => { + expect(addFailedCdn(['https://cdn-a.example.com'], 'https://cdn-b.example.com')).toEqual([ + 'https://cdn-a.example.com', + 'https://cdn-b.example.com', + ]); + }); + + it('is idempotent — re-adding a present CDN returns the same array reference', () => { + const failed = ['https://cdn-a.example.com']; + expect(addFailedCdn(failed, 'https://cdn-a.example.com')).toBe(failed); + }); +}); diff --git a/packages/spf/src/network/fetch.ts b/packages/spf/src/network/fetch.ts index 3561d297..6cc7d86a 100644 --- a/packages/spf/src/network/fetch.ts +++ b/packages/spf/src/network/fetch.ts @@ -124,6 +124,23 @@ export function getResponseText(response: ResponseLike): Promise { return response.text(); } +/** + * Fetch a resource and resolve its text body — the text analog of + * {@link FetchBytes}. A non-OK status rejects, so HTTP failures surface as + * rejections that callers (and decorators like the failover tracker) handle + * uniformly with network errors. + */ +export type FetchText = (addressable: Resource, options?: RequestInit) => Promise; + +/** Default {@link FetchText}: fetch the resource, reject on non-OK, return text. */ +export const fetchResolvableText: FetchText = async (addressable, options) => { + const response = await fetchResolvable(addressable, options); + if (!response.ok) { + throw new Error(`fetchResolvableText: ${response.status} ${response.statusText} for ${addressable.url}`); + } + return getResponseText(response); +}; + /** * Two-stage fetch helper: eagerly starts the HTTP request (TTFB is awaited), * then returns a lazy iterable over the response body. Separating connection diff --git a/packages/spf/src/playback/behaviors/resolve-cdn-priority.ts b/packages/spf/src/playback/behaviors/derive-cdn-priority.ts similarity index 86% rename from packages/spf/src/playback/behaviors/resolve-cdn-priority.ts rename to packages/spf/src/playback/behaviors/derive-cdn-priority.ts index ec530707..a54c7ac8 100644 --- a/packages/spf/src/playback/behaviors/resolve-cdn-priority.ts +++ b/packages/spf/src/playback/behaviors/derive-cdn-priority.ts @@ -29,9 +29,9 @@ import { defineBehavior } from '../../core/composition/create-composition'; import { createMachineReactor } from '../../core/reactors/create-machine-reactor'; import { computed, peek, type ReadonlySignal, type Signal } from '../../core/signals/primitives'; import { isResolvedPresentation, type MaybeResolvedPresentation } from '../../media/types'; -import { getOrderedCdnIds } from '../../media/utils/cdn'; +import { getCdnId as defaultGetCdnId, type GetCdnId, getOrderedCdnIds } from '../../media/utils/cdn'; -export interface ResolveCdnPriorityState { +export interface DeriveCdnPriorityState { presentation?: MaybeResolvedPresentation; cdnPriority?: string[]; } @@ -44,19 +44,22 @@ const samePriority = (a: string[] | undefined, b: string[]): boolean => * on src unload. * * @example - * const reactor = resolveCdnPriority.setup({ state }); + * const reactor = deriveCdnPriority.setup({ state }); */ -export const resolveCdnPriority = defineBehavior({ +export const deriveCdnPriority = defineBehavior({ stateKeys: ['presentation', 'cdnPriority'], contextKeys: [], setup: ({ state, + config = {}, }: { state: { - presentation: ReadonlySignal; - cdnPriority: Signal; + presentation: ReadonlySignal; + cdnPriority: Signal; }; + config?: { getCdnId?: GetCdnId }; }) => { + const getCdnId = config.getCdnId ?? defaultGetCdnId; const derivedStateSignal = computed(() => isResolvedPresentation(state.presentation.get()) ? ('presentation-resolved' as const) @@ -76,7 +79,7 @@ export const resolveCdnPriority = defineBehavior({ () => { const presentation = state.presentation.get(); if (!isResolvedPresentation(presentation)) return; - const next = getOrderedCdnIds(presentation); + const next = getOrderedCdnIds(presentation, getCdnId); // Skip the write when the CDN set is unchanged — a live reload swaps // in a new presentation object with the same hosts, and re-setting a // fresh array would re-fire the scope for an identical result. diff --git a/packages/spf/src/playback/behaviors/dom/load-segments.ts b/packages/spf/src/playback/behaviors/dom/load-segments.ts index b3a4c889..e417a723 100644 --- a/packages/spf/src/playback/behaviors/dom/load-segments.ts +++ b/packages/spf/src/playback/behaviors/dom/load-segments.ts @@ -56,7 +56,7 @@ import type { MaybeResolvedPresentation, Segment } from '../../../media/types'; import { findResolvedAudioTrack, findResolvedTextTrack, findResolvedVideoTrack } from '../../../media/utils/tracks'; import type { BufferState, SegmentLoaderActor, SourceBufferState } from '../../actors/dom/segment-loader'; import type { TextTrackSegmentLoaderActor } from '../../actors/text-track-segment-loader'; -import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../track-types'; +import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../../primitives/track-types'; // Re-export buffer state types for consumers that import them from this module. export type { BufferState, SourceBufferState }; diff --git a/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts b/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts index 3bcfdfa5..c536d318 100644 --- a/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts +++ b/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts @@ -65,6 +65,7 @@ import { createMachineReactor } from '../../../core/reactors/create-machine-reac import { computed, type ReadonlySignal, type Signal } from '../../../core/signals/primitives'; import { buildMimeCodec, createSourceBuffer } from '../../../media/dom/mse/mediasource-setup'; import type { MaybeResolvedPresentation, PartiallyResolvedTrack } from '../../../media/types'; +import type { GetCdnId } from '../../../media/utils/cdn'; import { getSelectedTrack, type TrackSelectionState } from '../../../media/utils/track-selection'; import { hasCodecs } from '../../../media/utils/tracks'; import type { BandwidthState } from '../../../network/bandwidth-estimator'; @@ -75,7 +76,8 @@ import { type SegmentLoaderActorConfig, } from '../../actors/dom/segment-loader'; import { createSourceBufferActor, type SourceBufferActor } from '../../actors/dom/source-buffer'; -import { AUDIO_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../track-types'; +import { failoverFetch } from '../../primitives/failover-fetch'; +import { AUDIO_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../../primitives/track-types'; /** * Media track type for MSE buffer setup. @@ -221,7 +223,7 @@ export const setupVideoBufferActors = defineBehavior({ bandwidthState: Signal; }; context: BufferActorsContextMap<'videoBufferActor', 'videoSegmentLoaderActor'>; - config?: object; + config?: SegmentLoaderActorConfig & { getCdnId?: GetCdnId }; }) => { // Bandwidth-sampling fetch. The factory accumulates EWMA state // internally; the callback bridges samples to engine state for ABR. @@ -238,10 +240,13 @@ export const setupVideoBufferActors = defineBehavior({ }, (next) => state.bandwidthState.set(next) ); + // Engine `config` layers over the per-type defaults; `failoverFetch` reads + // its `selectedKey` + `getCdnId` from the merged result. + const typeConfig = { ...VIDEO_TYPE_CONFIG, ...config }; return setupBufferActors({ state, context, - config: { ...VIDEO_TYPE_CONFIG, fetch: trackedFetch, ...config }, + config: { ...typeConfig, fetch: failoverFetch(trackedFetch, state, typeConfig) }, }); }, }); @@ -274,11 +279,14 @@ export const setupAudioBufferActors = defineBehavior({ }: { state: BufferActorsStateMap<'selectedAudioTrackId'>; context: BufferActorsContextMap<'audioBufferActor', 'audioSegmentLoaderActor'>; - config?: object; - }) => - setupBufferActors({ + config?: SegmentLoaderActorConfig & { getCdnId?: GetCdnId }; + }) => { + // Key order mirrors setupVideoBufferActors. + const typeConfig = { ...AUDIO_TYPE_CONFIG, ...config }; + return setupBufferActors({ state, context, - config: { ...AUDIO_TYPE_CONFIG, fetch: fetchStream, ...config }, - }), + config: { ...typeConfig, fetch: failoverFetch(fetchStream, state, typeConfig) }, + }); + }, }); diff --git a/packages/spf/src/playback/behaviors/dom/tsconfig.json b/packages/spf/src/playback/behaviors/dom/tsconfig.json index 71ba98dc..a115be83 100644 --- a/packages/spf/src/playback/behaviors/dom/tsconfig.json +++ b/packages/spf/src/playback/behaviors/dom/tsconfig.json @@ -12,6 +12,7 @@ { "path": "../../../network" }, { "path": "../../../media" }, { "path": "../../../media/dom" }, + { "path": "../../primitives" }, { "path": "../" }, { "path": "../../actors" }, { "path": "../../actors/dom" } diff --git a/packages/spf/src/playback/behaviors/resolve-track.ts b/packages/spf/src/playback/behaviors/resolve-track.ts index 757743b7..a49712b8 100644 --- a/packages/spf/src/playback/behaviors/resolve-track.ts +++ b/packages/spf/src/playback/behaviors/resolve-track.ts @@ -5,9 +5,11 @@ import { ConcurrentRunner, Task } from '../../core/tasks/task'; import { parseMediaPlaylist } from '../../media/hls/parse-media-playlist'; import type { MaybeResolvedPresentation, PartiallyResolvedTrack, ResolvedTrack } from '../../media/types'; import { isResolvedPresentation, isResolvedTrack } from '../../media/types'; +import type { GetCdnId } from '../../media/utils/cdn'; import { findTrack, updateTrackInPresentation } from '../../media/utils/tracks'; -import { fetchResolvable, getResponseText } from '../../network/fetch'; -import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from './track-types'; +import { fetchResolvableText as defaultFetchResolvableText, type FetchText } from '../../network/fetch'; +import { failoverFetch } from '../primitives/failover-fetch'; +import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../primitives/track-types'; // ============================================================================ // Specialization helper @@ -29,6 +31,7 @@ export interface ResolveTrackState { selectedVideoTrackId?: string; selectedAudioTrackId?: string; selectedTextTrackId?: string; + failedCdns?: string[]; } type SelectedTrackKey = 'selectedVideoTrackId' | 'selectedAudioTrackId' | 'selectedTextTrackId'; @@ -43,11 +46,22 @@ interface TrackResolutionConfig { presentation: MaybeResolvedPresentation, trackId: string ) => PartiallyResolvedTrack | ResolvedTrack | undefined; + /** Fetch a track's media-playlist text — already failover-decorated by the behavior. */ + fetchResolvableText?: FetchText; +} + +/** + * Engine-config slice each `resolve*` behavior reads to build its failover- + * decorated playlist fetch. + */ +interface ResolveTrackConfig { + /** CDN-id derivation for the failover trip; defaults to origin-based `getCdnId`. */ + getCdnId?: GetCdnId; } function setupTrackResolution({ state, - config: { selectedKey, findTrackToResolve }, + config: { selectedKey, findTrackToResolve, fetchResolvableText = defaultFetchResolvableText }, }: { state: ResolveTrackStateMap; config: TrackResolutionConfig; @@ -104,8 +118,11 @@ function setupTrackResolution({ // likely eventually passed down via config or a new "definitions" argument (CJP). new Task( async (signal) => { - const response = await fetchResolvable(track, { signal }); - const text = await getResponseText(response); + // `fetchResolvableText` is the behavior's failover-decorated + // fetch: it trips the CDN on a failed fetch (network error or + // non-OK status). A parse failure is a content issue, not a + // CDN-availability one, so it doesn't trip. + const text = await fetchResolvableText(track, { signal }); const mediaTrack = parseMediaPlaylist(text, track); // Updater handles undefined inputs by returning current @@ -163,11 +180,24 @@ const TEXT_TRACK_RESOLUTION_CONFIG = { export const resolveVideoTrack = defineBehavior({ stateKeys: ['presentation', 'selectedVideoTrackId'], contextKeys: [], - setup: ({ state, config = {} }: { state: ResolveTrackStateMap<'selectedVideoTrackId'>; config?: object }) => - setupTrackResolution({ + setup: ({ + state, + config = {}, + }: { + state: ResolveTrackStateMap<'selectedVideoTrackId'>; + config?: ResolveTrackConfig; + }) => { + // Engine `config` layers over the per-type defaults (mirrors the other + // per-type variants, see track-types.ts); `failoverFetch` reads its + // `selectedKey` + `getCdnId` from the merged result. `fetchResolvableText` + // is then placed AFTER the spread so the failover-decorated fetch wins — + // unlike segments, playlists expose no overridable per-type fetch. + const trackConfig = { ...VIDEO_TRACK_RESOLUTION_CONFIG, ...config }; + return setupTrackResolution({ state, - config: { ...VIDEO_TRACK_RESOLUTION_CONFIG, ...config }, - }), + config: { ...trackConfig, fetchResolvableText: failoverFetch(defaultFetchResolvableText, state, trackConfig) }, + }); + }, }); /** @@ -177,11 +207,20 @@ export const resolveVideoTrack = defineBehavior({ export const resolveAudioTrack = defineBehavior({ stateKeys: ['presentation', 'selectedAudioTrackId'], contextKeys: [], - setup: ({ state, config = {} }: { state: ResolveTrackStateMap<'selectedAudioTrackId'>; config?: object }) => - setupTrackResolution({ + setup: ({ + state, + config = {}, + }: { + state: ResolveTrackStateMap<'selectedAudioTrackId'>; + config?: ResolveTrackConfig; + }) => { + // Key order is load-bearing — see resolveVideoTrack. + const trackConfig = { ...AUDIO_TRACK_RESOLUTION_CONFIG, ...config }; + return setupTrackResolution({ state, - config: { ...AUDIO_TRACK_RESOLUTION_CONFIG, ...config }, - }), + config: { ...trackConfig, fetchResolvableText: failoverFetch(defaultFetchResolvableText, state, trackConfig) }, + }); + }, }); /** @@ -191,9 +230,18 @@ export const resolveAudioTrack = defineBehavior({ export const resolveTextTrack = defineBehavior({ stateKeys: ['presentation', 'selectedTextTrackId'], contextKeys: [], - setup: ({ state, config = {} }: { state: ResolveTrackStateMap<'selectedTextTrackId'>; config?: object }) => - setupTrackResolution({ + setup: ({ + state, + config = {}, + }: { + state: ResolveTrackStateMap<'selectedTextTrackId'>; + config?: ResolveTrackConfig; + }) => { + // Key order is load-bearing — see resolveVideoTrack. + const trackConfig = { ...TEXT_TRACK_RESOLUTION_CONFIG, ...config }; + return setupTrackResolution({ state, - config: { ...TEXT_TRACK_RESOLUTION_CONFIG, ...config }, - }), + config: { ...trackConfig, fetchResolvableText: failoverFetch(defaultFetchResolvableText, state, trackConfig) }, + }); + }, }); diff --git a/packages/spf/src/playback/behaviors/select-tracks.ts b/packages/spf/src/playback/behaviors/select-tracks.ts index 18b181e3..06743241 100644 --- a/packages/spf/src/playback/behaviors/select-tracks.ts +++ b/packages/spf/src/playback/behaviors/select-tracks.ts @@ -44,7 +44,7 @@ import { type VideoSelectionConfig, } from '../../media/primitives/select-tracks'; import { isResolvedPresentation } from '../../media/types'; -import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from './track-types'; +import { AUDIO_TYPE_CONFIG, TEXT_TYPE_CONFIG, VIDEO_TYPE_CONFIG } from '../primitives/track-types'; // ============================================================================ // Specialization helper diff --git a/packages/spf/src/playback/behaviors/setup-failover-monitor.ts b/packages/spf/src/playback/behaviors/setup-failover-monitor.ts new file mode 100644 index 00000000..fae53f9c --- /dev/null +++ b/packages/spf/src/playback/behaviors/setup-failover-monitor.ts @@ -0,0 +1,113 @@ +/** + * **CDN failover cooldown.** The expiry half of multi-CDN failover. Fetch sites + * own the *trip*: on a failed fetch they add the failing CDN (origin) to the + * `failedCdns` state signal directly. This behavior owns the *expiry*: while a + * presentation is resolved, it watches `failedCdns` and, for each CDN that + * appears, schedules a timer to remove it once its cooldown lapses. + * `track-switching`'s `excludeFailedCdns` constraint prunes a failed CDN's + * tracks and the active-CDN scope falls to the next one — and back, once the + * cooldown removes it here. + * + * Lifecycle is per-source: timers + `failedCdns` are cleared on exit (a new + * source starts with a clean slate). Policy (cooldown) is engine config. This is + * the minimal `network-resilience` slice — a single failure trips a CDN, since + * transient blips are the retry layer's job (it sits below the fetch sites, so + * anything that reaches `failedCdns` is already terminal). + */ + +import { defineBehavior } from '../../core/composition/create-composition'; +import { createMachineReactor } from '../../core/reactors/create-machine-reactor'; +import { computed, type ReadonlySignal, type Signal, update } from '../../core/signals/primitives'; +import { isResolvedPresentation, type MaybeResolvedPresentation } from '../../media/types'; + +/** + * Failover policy: how long a CDN stays excluded after a failed fetch trips it. + * Supplied via engine config. + */ +export interface FailoverMonitorConfig { + /** How long a tripped CDN stays excluded, in milliseconds. */ + cooldownMs: number; +} + +export const DEFAULT_FAILOVER_MONITOR_CONFIG: FailoverMonitorConfig = { + // 5 minutes — a CDN outage is an infrastructure problem that outlasts a + // transient blip, so re-probing it sooner mostly re-trips. Matches the + // prevailing prior-art default (ExoPlayer's location exclusion, hls.js's + // content-steering penalty box). + cooldownMs: 300_000, +}; + +export interface SetupFailoverMonitorState { + presentation?: MaybeResolvedPresentation; + failedCdns?: string[]; +} + +export interface SetupFailoverMonitorConfig { + /** Failover policy (cooldown); defaults to `DEFAULT_FAILOVER_MONITOR_CONFIG`. */ + failover?: Partial; +} + +/** + * Expire failed CDNs from `failedCdns` once their cooldown lapses, for the + * resolved source. + * + * @example + * const reactor = setupFailoverMonitor.setup({ state }); + */ +export const setupFailoverMonitor = defineBehavior({ + stateKeys: ['presentation', 'failedCdns'], + contextKeys: [], + setup: ({ + state, + config = {}, + }: { + state: { + presentation: ReadonlySignal; + failedCdns: Signal; + }; + config?: SetupFailoverMonitorConfig; + }) => { + const cooldownMs = config.failover?.cooldownMs ?? DEFAULT_FAILOVER_MONITOR_CONFIG.cooldownMs; + // CDN id → its pending cooldown-removal timer. Shared by the `effects` + // scheduler (adds a timer per newly-failed CDN) and the exit cleanup + // (clears them). Per-source: emptied on exit, so it re-enters clean. + const timers = new Map>(); + const derivedStateSignal = computed(() => + isResolvedPresentation(state.presentation.get()) + ? ('presentation-resolved' as const) + : ('presentation-unresolved' as const) + ); + + return createMachineReactor({ + initial: 'presentation-unresolved', + monitor: () => derivedStateSignal.get(), + states: { + 'presentation-unresolved': {}, + 'presentation-resolved': { + // Cleanup-binds-to-setup: on exit (src unload + destroy) clear the + // pending timers and reset `failedCdns` for the next source. + entry: () => () => { + timers.forEach((timer) => clearTimeout(timer)); + timers.clear(); + state.failedCdns.set(undefined); + }, + effects: [ + () => { + const failed = state.failedCdns.get() ?? []; + failed.forEach((cdn) => { + // Idempotent: a CDN already counting down keeps its original + // deadline (re-failing it mid-cooldown doesn't extend it). + if (timers.has(cdn)) return; + const timer = setTimeout(() => { + timers.delete(cdn); + update(state.failedCdns, (current) => current?.filter((c) => c !== cdn)); + }, cooldownMs); + timers.set(cdn, timer); + }); + }, + ], + }, + }, + }); + }, +}); diff --git a/packages/spf/src/playback/behaviors/tests/resolve-cdn-priority.test.ts b/packages/spf/src/playback/behaviors/tests/derive-cdn-priority.test.ts similarity index 86% rename from packages/spf/src/playback/behaviors/tests/resolve-cdn-priority.test.ts rename to packages/spf/src/playback/behaviors/tests/derive-cdn-priority.test.ts index 095d9b18..ff22bb33 100644 --- a/packages/spf/src/playback/behaviors/tests/resolve-cdn-priority.test.ts +++ b/packages/spf/src/playback/behaviors/tests/derive-cdn-priority.test.ts @@ -2,9 +2,9 @@ import { describe, expect, it } from 'vitest'; import type { StateSignals } from '../../../core/composition/create-composition'; import { signal } from '../../../core/signals/primitives'; import type { MaybeResolvedPresentation, PartiallyResolvedVideoTrack, Presentation } from '../../../media/types'; -import { type ResolveCdnPriorityState, resolveCdnPriority } from '../resolve-cdn-priority'; +import { type DeriveCdnPriorityState, deriveCdnPriority } from '../derive-cdn-priority'; -function makeState(initial: Partial = {}): StateSignals { +function makeState(initial: Partial = {}): StateSignals { return { presentation: signal(initial.presentation), cdnPriority: signal(initial.cdnPriority), @@ -50,10 +50,10 @@ const redundant = (id = 'pres-1'): Presentation => const flush = () => Promise.resolve().then(() => Promise.resolve()); -describe('resolveCdnPriority', () => { +describe('deriveCdnPriority', () => { it('does nothing without a presentation', async () => { const state = makeState(); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); expect(state.cdnPriority.get()).toBeUndefined(); reactor.destroy(); @@ -61,7 +61,7 @@ describe('resolveCdnPriority', () => { it('publishes the manifest-ordered CDN list on src load', async () => { const state = makeState({ presentation: redundant() }); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']); reactor.destroy(); @@ -69,7 +69,7 @@ describe('resolveCdnPriority', () => { it('publishes a single-entry list for a non-redundant source', async () => { const state = makeState({ presentation: presentationWith(['https://cdn-a.example.com/720p.m3u8']) }); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com']); reactor.destroy(); @@ -77,7 +77,7 @@ describe('resolveCdnPriority', () => { it('does not re-set the list when a resolved swap keeps the same CDNs', async () => { const state = makeState({ presentation: redundant() }); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); const first = state.cdnPriority.get(); @@ -92,7 +92,7 @@ describe('resolveCdnPriority', () => { it('updates the list when a resolved swap changes the CDN order', async () => { const state = makeState({ presentation: redundant() }); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']); @@ -107,7 +107,7 @@ describe('resolveCdnPriority', () => { it('clears cdnPriority on src unload', async () => { const state = makeState({ presentation: redundant() }); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); expect(state.cdnPriority.get()).toBeDefined(); @@ -120,7 +120,7 @@ describe('resolveCdnPriority', () => { it('clears cdnPriority on destroy', async () => { const state = makeState({ presentation: redundant() }); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); expect(state.cdnPriority.get()).toBeDefined(); @@ -130,7 +130,7 @@ describe('resolveCdnPriority', () => { it('re-publishes after a src reset (undefined → new resolved)', async () => { const state = makeState({ presentation: redundant() }); - const reactor = resolveCdnPriority.setup({ state }); + const reactor = deriveCdnPriority.setup({ state }); await flush(); expect(state.cdnPriority.get()).toEqual(['https://cdn-a.example.com', 'https://cdn-b.example.com']); diff --git a/packages/spf/src/playback/behaviors/tests/resolve-track.test.ts b/packages/spf/src/playback/behaviors/tests/resolve-track.test.ts index 85a25a81..b1b500af 100644 --- a/packages/spf/src/playback/behaviors/tests/resolve-track.test.ts +++ b/packages/spf/src/playback/behaviors/tests/resolve-track.test.ts @@ -21,6 +21,7 @@ function makeState(initial: ResolveTrackState = {}): StateSignals(initial.selectedVideoTrackId), selectedAudioTrackId: signal(initial.selectedAudioTrackId), selectedTextTrackId: signal(initial.selectedTextTrackId), + failedCdns: signal(initial.failedCdns), }; } diff --git a/packages/spf/src/playback/behaviors/tests/setup-failover-monitor.test.ts b/packages/spf/src/playback/behaviors/tests/setup-failover-monitor.test.ts new file mode 100644 index 00000000..93df746b --- /dev/null +++ b/packages/spf/src/playback/behaviors/tests/setup-failover-monitor.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { signal } from '../../../core/signals/primitives'; +import type { MaybeResolvedPresentation, Presentation } from '../../../media/types'; +import { DEFAULT_FAILOVER_MONITOR_CONFIG, setupFailoverMonitor } from '../setup-failover-monitor'; + +const resolved = (): Presentation => + ({ id: 'pres-1', url: 'https://cdn-a.example.com/master.m3u8', startTime: 0, selectionSets: [] }) as Presentation; + +const makeState = (presentation?: MaybeResolvedPresentation) => ({ + presentation: signal(presentation), + failedCdns: signal(undefined), +}); + +const flush = () => Promise.resolve().then(() => Promise.resolve()); + +const A = 'https://cdn-a.example.com'; +const B = 'https://cdn-b.example.com'; + +describe('setupFailoverMonitor', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(0); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('removes a failed CDN once its cooldown lapses', async () => { + const state = makeState(resolved()); + const reactor = setupFailoverMonitor.setup({ state, config: { failover: { cooldownMs: 1000 } } }); + await flush(); + + state.failedCdns.set([A]); // a fetch site tripped cdn-a + await flush(); + expect(state.failedCdns.get()).toEqual([A]); + + await vi.advanceTimersByTimeAsync(1000); + expect(state.failedCdns.get()).toEqual([]); + + reactor.destroy(); + }); + + it('expires each CDN on its own cooldown', async () => { + const state = makeState(resolved()); + const reactor = setupFailoverMonitor.setup({ state, config: { failover: { cooldownMs: 1000 } } }); + await flush(); + + state.failedCdns.set([A]); + await flush(); + await vi.advanceTimersByTimeAsync(600); + + state.failedCdns.set([A, B]); // cdn-b tripped 600ms after cdn-a + await flush(); + + await vi.advanceTimersByTimeAsync(400); // cdn-a's cooldown lapses (t=1000); cdn-b's (t=1600) not yet + expect(state.failedCdns.get()).toEqual([B]); + + await vi.advanceTimersByTimeAsync(600); // cdn-b's cooldown lapses (t=1600) + expect(state.failedCdns.get()).toEqual([]); + + reactor.destroy(); + }); + + it('clears failedCdns and pending timers on src unload', async () => { + const state = makeState(resolved()); + const reactor = setupFailoverMonitor.setup({ state, config: { failover: { cooldownMs: 1000 } } }); + await flush(); + + state.failedCdns.set([A]); + await flush(); + expect(state.failedCdns.get()).toEqual([A]); + + state.presentation.set(undefined); + await flush(); + expect(state.failedCdns.get()).toBeUndefined(); + + reactor.destroy(); + }); + + it('exposes a sensible failover default', () => { + expect(DEFAULT_FAILOVER_MONITOR_CONFIG.cooldownMs).toBeGreaterThan(0); + }); +}); diff --git a/packages/spf/src/playback/behaviors/tests/track-switching.test.ts b/packages/spf/src/playback/behaviors/tests/track-switching.test.ts index db087dc7..0c88a5ff 100644 --- a/packages/spf/src/playback/behaviors/tests/track-switching.test.ts +++ b/packages/spf/src/playback/behaviors/tests/track-switching.test.ts @@ -12,6 +12,7 @@ import type { } from '../../../media/types'; import type { BandwidthState } from '../../../network/bandwidth-estimator'; import { + applyConstraints, applyRules, type SelectionRule, type SwitchVideoTrackConfig, @@ -735,6 +736,43 @@ describe('applyRules', () => { }); }); +// ============================================================================ +// applyConstraints — the hard-constraints pre-pass (pure; no signals) +// ============================================================================ + +describe('applyConstraints', () => { + const track = (id: string) => ({ id }); + const all = [track('a'), track('b'), track('c')]; + const noDeps = { state: {}, context: {}, config: {} }; + + const noA: SelectionRule<{ id: string }> = (tracks) => tracks.filter((t) => t.id !== 'a'); + const noC: SelectionRule<{ id: string }> = (tracks) => tracks.filter((t) => t.id !== 'c'); + + it('removes what each constraint excludes (pooled)', () => { + expect(applyConstraints([noA, noC], all, noDeps).map((t) => t.id)).toEqual(['b']); + }); + + it('is order-independent', () => { + expect(applyConstraints([noA, noC], all, noDeps)).toEqual(applyConstraints([noC, noA], all, noDeps)); + }); + + it('preserves an empty result — no fall-through, unlike applyRules', () => { + const none: SelectionRule<{ id: string }> = () => []; + expect(applyConstraints([none], all, noDeps)).toEqual([]); + }); + + it('runs every constraint — no early-bail at a single survivor', () => { + const toA: SelectionRule<{ id: string }> = (tracks) => tracks.filter((t) => t.id === 'a'); + let laterCalled = false; + const later: SelectionRule<{ id: string }> = (tracks) => { + laterCalled = true; + return tracks; + }; + applyConstraints([toA, later], all, noDeps); + expect(laterCalled).toBe(true); + }); +}); + // ============================================================================ // preferActiveCdn — active-CDN scope (shared by video + audio) // ============================================================================ @@ -813,8 +851,8 @@ describe('preferActiveCdn (active-CDN scope)', () => { }); it('applies the scope when cdnPriority arrives after the first pick (composition-order independence)', async () => { - // Guards against the pick depending on `resolveCdnPriority` being composed - // *before* `switchVideoTrack`. The worst case — resolveCdnPriority last — is + // Guards against the pick depending on `deriveCdnPriority` being composed + // *before* `switchVideoTrack`. The worst case — deriveCdnPriority last — is // equivalent to cdnPriority being written after switch*'s first pick. The // scope subscribes to cdnPriority even while it's undefined, so a late write // must re-fire and correct the pick. @@ -863,3 +901,87 @@ describe('preferActiveCdn (active-CDN scope)', () => { reactor.destroy(); }); }); + +// ============================================================================ +// excludeFailedCdns — the failover constraint (hard pre-pass) + scope interplay +// ============================================================================ + +describe('excludeFailedCdns (failover constraint)', () => { + const cdnVideoTrack = (id: string, host: string, bandwidth: number): PartiallyResolvedVideoTrack => ({ + type: 'video', + codecs: [], + id, + url: `https://${host}/${id}.m3u8`, + bandwidth, + mimeType: 'video/mp4', + }); + + const multiCdn = () => + createPresentation([ + cdnVideoTrack('720p-a', 'cdn-a.example.com', 2_400_000), + cdnVideoTrack('720p-b', 'cdn-b.example.com', 2_400_000), + cdnVideoTrack('1080p-a', 'cdn-a.example.com', 4_800_000), + cdnVideoTrack('1080p-b', 'cdn-b.example.com', 4_800_000), + ]); + + const makeState = (failedCdns?: string[]) => ({ + presentation: signal(multiCdn()), + bandwidthState: signal(createBandwidthState(10_000_000)), + selectedVideoTrackId: signal(undefined), + userVideoTrackSelection: signal | undefined>(undefined), + cdnPriority: signal(['https://cdn-a.example.com', 'https://cdn-b.example.com']), + failedCdns: signal(failedCdns), + }); + + it('excludes nothing when failedCdns is absent — picks the primary', async () => { + const state = makeState(undefined); + const reactor = switchVideoTrack.setup({ state }); + await flush(); + expect(state.selectedVideoTrackId.get()).toBe('1080p-a'); + reactor.destroy(); + }); + + it('fails over to the next CDN when the primary is in cooldown', async () => { + const state = makeState(['https://cdn-a.example.com']); + const reactor = switchVideoTrack.setup({ state }); + await flush(); + // cdn-a's tracks are pruned by the constraint, so the scope falls to cdn-b. + expect(state.selectedVideoTrackId.get()).toBe('1080p-b'); + reactor.destroy(); + }); + + it('fails over reactively, then returns to the primary on recovery', async () => { + const state = makeState(undefined); + const reactor = switchVideoTrack.setup({ state }); + await flush(); + expect(state.selectedVideoTrackId.get()).toBe('1080p-a'); + + // cdn-a enters cooldown → prune → scope falls to cdn-b. + state.failedCdns.set(['https://cdn-a.example.com']); + await flush(); + expect(state.selectedVideoTrackId.get()).toBe('1080p-b'); + + // cdn-a recovers → its tracks reappear → scope snaps back to the primary. + state.failedCdns.set([]); + await flush(); + expect(state.selectedVideoTrackId.get()).toBe('1080p-a'); + + reactor.destroy(); + }); + + it('keeps the prior pick when every CDN is in cooldown (nothing playable)', async () => { + const state = makeState(undefined); + const reactor = switchVideoTrack.setup({ state }); + await flush(); + expect(state.selectedVideoTrackId.get()).toBe('1080p-a'); + + // All CDNs cooled down → constraints prune everything → no playable set → + // the effect no-ops, leaving the last pick in place (deferred terminal-state + // modeling). + state.failedCdns.set(['https://cdn-a.example.com', 'https://cdn-b.example.com']); + await flush(); + expect(state.selectedVideoTrackId.get()).toBe('1080p-a'); + + reactor.destroy(); + }); +}); diff --git a/packages/spf/src/playback/behaviors/track-switching.ts b/packages/spf/src/playback/behaviors/track-switching.ts index 0cfe2cba..6ed8cf82 100644 --- a/packages/spf/src/playback/behaviors/track-switching.ts +++ b/packages/spf/src/playback/behaviors/track-switching.ts @@ -4,17 +4,20 @@ * default, react to user intent and algorithmic ranking, and clear it on src * unload. * - * Selection runs a small ordered chain of rules over the candidate tracks - * (`applyRules`). Each rule narrows or reorders the list and reads the signals - * it needs at apply time, so the effect subscribes to exactly what the applied - * rules consult. Today the chain is three rules, most authoritative first: + * Selection runs in two stages. First a **hard-constraints pre-pass** + * (`applyConstraints`) prunes the unplayable from the candidate set — today the + * failed-CDN constraint (`excludeFailedCdns`, failover cooldown); capability + * probing will join it. Then a small ordered chain of rules (`applyRules`) picks + * among the survivors. Each constraint/rule reads the signals it needs at apply + * time, so the effect subscribes to exactly what was consulted. The chain is + * three rules, most authoritative first: * * 1. **user intent** — a soft filter on `user*TrackSelection`: narrow to the * partial-track match; an empty match falls through to the full set. * 2. **active CDN** — a soft filter on `cdnPriority` (`preferActiveCdn`): * narrow to the highest-priority CDN that still has tracks; an empty match * falls through. Shared by video and audio, so every type stays on one CDN - * (`resolveCdnPriority` owns the list). No-op for non-redundant sources. + * (`deriveCdnPriority` owns the list). No-op for non-redundant sources. * 3. **ranking** — the terminal sort: `rankByBandwidth`, shared by video and * audio. Fitting tracks (within the throughput threshold) first, highest * bitrate first; over-throughput tracks after, least-over first. Hysteresis @@ -30,17 +33,17 @@ * (canonical cleanup-binds-to-setup per `reactors.md`). * * The pick is the head of the chain's result (`applyRules(...)[0]`). Each - * variant supplies its **rule chain** via config; `setupTrackSwitching` owns - * only the lifecycle and runs whatever chain it's given. Both variants today run + * variant supplies its **constraints + rule chain** via config; + * `setupTrackSwitching` owns only the lifecycle and runs what it's given. Both + * variants today run constraints `[excludeFailedCdns]` then rules * `[filterByUserSelection, preferActiveCdn, rankByBandwidth]`; `switchVideoTrack` - * also accepts ABR tuning config, `switchAudioTrack` takes none. + * also accepts ABR tuning config, `switchAudioTrack` takes none. (The active-CDN + * *scope* is the sticky-pick half of multi-CDN; the failed-CDN *constraint* is + * the failover half — prune the cooled-down CDN, the scope falls to the next.) * - * Deferred (not yet in the chain): a hard-constraints pre-pass (capability - * probing, CDN *failover* — excluding a failed CDN's tracks during cooldown) - * gating the candidate set, and audio's preferred-language / default-track - * selection as standing soft-filter rules — previously the empty-slot picker, - * dropped in the move to the rule chain. (The active-CDN *scope* above is the - * sticky-pick half of multi-CDN; the failed-CDN constraint is the failover half.) + * Deferred: capability probing as a second constraint; audio's preferred- + * language / default-track selection as standing soft-filter rules (previously + * the empty-slot picker, dropped in the move to the rule chain). */ import { type AnySlotMap, defineBehavior } from '../../core/composition/create-composition'; @@ -56,7 +59,7 @@ import { type PartiallyResolvedVideoTrack, type VideoTrack, } from '../../media/types'; -import { getCdnId } from '../../media/utils/cdn'; +import { getCdnId as defaultGetCdnId, type GetCdnId } from '../../media/utils/cdn'; import { getTracksByType } from '../../media/utils/tracks'; import type { BandwidthConfig, BandwidthState } from '../../network/bandwidth-estimator'; import { DEFAULT_BANDWIDTH_CONFIG, getBandwidthEstimate } from '../../network/bandwidth-estimator'; @@ -91,6 +94,8 @@ export interface SwitchVideoTrackConfig { quality?: Partial; bandwidth?: Partial; initialBandwidth?: number; + /** Override CDN-id derivation (shared by the CDN scope + failover constraint). */ + getCdnId?: GetCdnId; } /** Default initial-bandwidth value before bandwidth measurements arrive. */ @@ -153,6 +158,31 @@ export function applyRules( return current; } +/** + * Apply hard constraints to a candidate list — the pre-pass that runs before the + * rule chain. A constraint shares a rule's signature but its exclusion is + * *hard*: it removes the unplayable (a codec the environment can't decode, a CDN + * in failover cooldown) and a removed track is never attempted. Unlike + * `applyRules`, this never skips an empty result and never early-bails — every + * constraint always applies, and an empty survivor set is a real outcome + * ("nothing playable here"), not a fall-through. Because each constraint only + * removes, the order they run in can't change the result. + * + * @param constraints - Constraints to apply (pooled, order-independent) + * @param tracks - Candidate tracks + * @param deps - The behavior's `{ state, context, config }`, passed to each constraint + * @returns The playable survivors (possibly empty) + */ +export function applyConstraints( + constraints: readonly SelectionRule[], + tracks: readonly T[], + deps: SelectionRuleDeps +): readonly T[] { + let current = tracks; + for (const constraint of constraints) current = constraint(current, deps); + return current; +} + // ============================================================================ // Specialization helper // @@ -216,16 +246,19 @@ type TrackSwitchingStateMap = { /** * Config `setupTrackSwitching` itself reads — its own wiring: which selection * slot to write and clear (`selectionKey`), how to enumerate candidate tracks - * (`getTracks`), and the **rule chain** to run (`rules`). Rule-specific config - * is deliberately absent — each rule declares the fields it reads as *optional* - * on its own config view (`UserSelectionConfig`, `BandwidthRankerConfig`), so - * the behavior never enumerates a rule's config. The variant builds the - * concrete config as this base plus whatever its chain's rules consult; it - * flows through untouched as the `C` type param on `setupTrackSwitching`. + * (`getTracks`), the optional **hard-constraints pre-pass** (`constraints`, + * applied before the chain to prune the unplayable), and the **rule chain** to + * run (`rules`). Rule-/constraint-specific config is deliberately absent — each + * declares the fields it reads as *optional* on its own config view + * (`UserSelectionConfig`, `BandwidthRankerConfig`), so the behavior never + * enumerates them. The variant builds the concrete config as this base plus + * whatever its chain consults; it flows through untouched as the `C` type param + * on `setupTrackSwitching`. */ interface TrackSwitchingConfig { selectionKey: S; getTracks: (presentation: MaybeResolvedPresentation) => readonly T[]; + constraints?: readonly SelectionRule, AnySlotMap, TrackSwitchingConfig>[]; rules: readonly SelectionRule, AnySlotMap, TrackSwitchingConfig>[]; } @@ -277,7 +310,7 @@ type BandwidthRankerConfig = /** * State the active-CDN scope reads: the lifecycle map plus an *optional* * `cdnPriority` — the manifest-ordered CDN list (most-preferred first). The - * signal exists only when the composition includes `resolveCdnPriority` (which + * signal exists only when the composition includes `deriveCdnPriority` (which * materializes + owns it); the scope reads it defensively and passes through * when it's absent (no CDN preference). */ @@ -285,6 +318,27 @@ type CdnScopeStateMap = TrackSwitchingStateMap & { cdnPriority?: ReadonlySignal; }; +/** + * State the failed-CDN constraint reads: the lifecycle map plus an *optional* + * `failedCdns` — the CDN ids currently in failover cooldown. The signal exists + * only when the composition includes a failover monitor (or an external driver); the + * constraint reads it defensively and excludes nothing when it's absent. + */ +type CdnConstraintStateMap = TrackSwitchingStateMap & { + failedCdns?: ReadonlySignal; +}; + +/** + * Config the CDN rules read: the base config plus an *optional* `getCdnId` + * override. Both `excludeFailedCdns` and `preferActiveCdn` derive a track's CDN + * from its URL; the override must be the *same* one `deriveCdnPriority` and the + * failover trip use, or the keys stop matching. Optional → defaults to the + * origin-based `getCdnId`, so the base config (without it) stays assignable. + */ +type CdnRuleConfig = TrackSwitchingConfig & { + getCdnId?: GetCdnId; +}; + type VideoTrackCandidate = PartiallyResolvedVideoTrack | VideoTrack; type AudioTrackCandidate = PartiallyResolvedAudioTrack | AudioTrack; @@ -310,9 +364,32 @@ function filterByUserSelection matchesPartialTrack(track, filter)) : tracks; } +/** + * Failed-CDN constraint — a *hard* filter (constraints pre-pass), shared by + * video and audio. Removes tracks served from a CDN currently in failover + * cooldown (`failedCdns`, written by the failover monitor). Removed tracks are never + * attempted; the scope then narrows to the next surviving CDN in `cdnPriority`, + * and snaps back to the primary once it leaves cooldown. + * + * Passes everything through when there's no `failedCdns` signal/value. When it + * prunes *every* track (all CDNs cooled down), the empty result is preserved + * (per `applyConstraints`) — "nothing playable," which today leaves the prior + * pick in place. + */ +function excludeFailedCdns( + tracks: readonly T[], + { state, config }: SelectionRuleDeps, AnySlotMap, CdnRuleConfig> +): readonly T[] { + const failed = state.failedCdns?.get(); + if (!failed?.length) return tracks; + const getCdnId = config.getCdnId ?? defaultGetCdnId; + const failedSet = new Set(failed); + return tracks.filter((track) => !failedSet.has(getCdnId(track.url))); +} + /** * Active-CDN scope — a soft filter, shared by video and audio. Narrows to the - * highest-priority CDN in `cdnPriority` (owned by `resolveCdnPriority`) that + * highest-priority CDN in `cdnPriority` (owned by `deriveCdnPriority`) that * still has tracks, so every track type stays on one CDN. A redundant-streams * source lists the same renditions on multiple hosts; this keeps the pick on one * host rather than letting the ranker drift across them. @@ -327,15 +404,17 @@ function filterByUserSelection( tracks: readonly T[], - { state }: SelectionRuleDeps, AnySlotMap, TrackSwitchingConfig> + { state, config }: SelectionRuleDeps, AnySlotMap, CdnRuleConfig> ): readonly T[] { const cdnPriority = state.cdnPriority?.get(); if (!cdnPriority?.length) return tracks; + const getCdnId = config.getCdnId ?? defaultGetCdnId; for (const cdn of cdnPriority) { const tracksUsingCdn = tracks.filter((track) => getCdnId(track.url) === cdn); if (tracksUsingCdn.length) return tracksUsingCdn; @@ -416,15 +495,13 @@ function setupTrackSwitching< ); // The playable candidate set — the tracks the rule chain gets to pick from, - // derived *outside* the reaction. This is the seam a future hard-constraints - // pre-pass (capability probing, CDN failover) occupies: it would narrow these - // tracks before the chain runs — - // isResolvedPresentation(p) ? applyConstraints(constraints, getTracks(p), deps) : [] - // — and because it's a `computed`, the constraints' own signal reads are - // tracked here. The effect reads it with `.get()`, so when the playable set - // changes — a new source, or a *dynamic* constraint like a CDN entering - // cooldown — the effect re-picks. Today it's just the type's tracks while a - // presentation is resolved. + // derived *outside* the reaction. The hard-constraints pre-pass (capability + // probing, CDN-failover cooldown) narrows the type's tracks before the chain + // runs. Because this is a `computed`, a constraint's own signal reads (e.g. + // `cdnHealth`) are tracked here, so when the playable set changes — a new + // source, or a *dynamic* constraint like a CDN entering cooldown — the effect + // re-picks. With no constraints configured this is just the type's tracks + // while a presentation is resolved. // // The `equals` gates notification on the *set of track ids*, not array // identity: a live playlist refresh swaps in a new presentation object with @@ -436,7 +513,8 @@ function setupTrackSwitching< const candidateSet = computed( () => { const presentation = state.presentation.get(); - return isResolvedPresentation(presentation) ? getTracks(presentation) : []; + if (!isResolvedPresentation(presentation)) return []; + return applyConstraints(config.constraints ?? [], getTracks(presentation), deps); }, { equals: (a, b) => a.length === b.length && a.every((track) => b.some((other) => other.id === track.id)) } ); @@ -524,6 +602,7 @@ export const switchVideoTrack = defineBehavior({ selectionKey: 'selectedVideoTrackId', userSelectionKey: 'userVideoTrackSelection', getTracks: (presentation) => getTracksByType(presentation, 'video') as readonly VideoTrackCandidate[], + constraints: [excludeFailedCdns], rules: [filterByUserSelection, preferActiveCdn, rankByBandwidth], }, }), @@ -549,14 +628,32 @@ export const switchVideoTrack = defineBehavior({ export const switchAudioTrack = defineBehavior({ stateKeys: ['presentation', 'selectedAudioTrackId'], contextKeys: [], - setup: ({ state, ...otherProps }: { state: TrackSwitchingStateMap<'selectedAudioTrackId'> }) => + setup: ({ + state, + config, + ...otherProps + }: { + state: TrackSwitchingStateMap<'selectedAudioTrackId'>; + // Shares the video config shape so the engine config spreads through (CDN + // derivation + any future cross-cutting fields). + config?: SwitchVideoTrackConfig; + }) => setupTrackSwitching({ ...otherProps, state, config: { + // Spread engine config so cross-cutting fields (`getCdnId`, future shared + // tuning) flow through like they do for video, then override the per-type + // wiring. Video-only ABR tuning (`quality`/`bandwidth`/`initialBandwidth`) + // rides along into the shared `rankByBandwidth` too; harmless since audio + // has no `bandwidthState` to act on it and the ranker always yields a pick. + // FOLLOW-UP: a shared config type for the genuinely cross-cutting fields + // would keep video-only tuning out of audio entirely (CJP). + ...config, selectionKey: 'selectedAudioTrackId', userSelectionKey: 'userAudioTrackSelection', getTracks: (presentation) => getTracksByType(presentation, 'audio') as readonly AudioTrackCandidate[], + constraints: [excludeFailedCdns], rules: [filterByUserSelection, preferActiveCdn, rankByBandwidth], }, }), diff --git a/packages/spf/src/playback/behaviors/tsconfig.json b/packages/spf/src/playback/behaviors/tsconfig.json index 3ba6006f..1ebbfaf7 100644 --- a/packages/spf/src/playback/behaviors/tsconfig.json +++ b/packages/spf/src/playback/behaviors/tsconfig.json @@ -11,6 +11,7 @@ { "path": "../../core" }, { "path": "../../network" }, { "path": "../../media" }, + { "path": "../primitives" }, { "path": "../actors" } ], "include": ["./*.ts", "./tests/**/*.ts"], diff --git a/packages/spf/src/playback/engines/hls/engine-audio-only.ts b/packages/spf/src/playback/engines/hls/engine-audio-only.ts index d3fb2384..77fb0a4f 100644 --- a/packages/spf/src/playback/engines/hls/engine-audio-only.ts +++ b/packages/spf/src/playback/engines/hls/engine-audio-only.ts @@ -9,6 +9,7 @@ import type { BackBufferConfig } from '../../../media/buffer/back-buffer'; import type { ForwardBufferConfig } from '../../../media/buffer/forward-buffer'; import { parseMultivariantPlaylist } from '../../../media/hls/parse-multivariant'; import type { AudioTrack, MaybeResolvedPresentation } from '../../../media/types'; +import type { GetCdnId } from '../../../media/utils/cdn'; import { getResolvedSelectedTrackDuration } from '../../../media/utils/track-selection'; import type { SegmentLoaderActor } from '../../actors/dom/segment-loader'; import type { SourceBufferActor } from '../../actors/dom/source-buffer'; @@ -16,6 +17,7 @@ import { calculatePresentationDuration, type PresentationDurationResolver, } from '../../behaviors/calculate-presentation-duration'; +import { deriveCdnPriority } from '../../behaviors/derive-cdn-priority'; import { endOfStream } from '../../behaviors/dom/end-of-stream'; import { loadAudioSegments } from '../../behaviors/dom/load-segments'; import { setupAudioBufferActors } from '../../behaviors/dom/setup-buffer-actors'; @@ -23,9 +25,9 @@ import { setupMediaSource } from '../../behaviors/dom/setup-mediasource'; import { trackCurrentTime } from '../../behaviors/dom/track-current-time'; import { trackLoadTriggers } from '../../behaviors/dom/track-load-triggers'; import { updateMediaSourceDuration } from '../../behaviors/dom/update-mediasource-duration'; -import { resolveCdnPriority } from '../../behaviors/resolve-cdn-priority'; import { type ParsePresentation, resolvePresentation } from '../../behaviors/resolve-presentation'; import { resolveAudioTrack } from '../../behaviors/resolve-track'; +import { type FailoverMonitorConfig, setupFailoverMonitor } from '../../behaviors/setup-failover-monitor'; import { syncPreload } from '../../behaviors/sync-preload'; import { switchAudioTrack } from '../../behaviors/track-switching'; @@ -54,11 +56,17 @@ export interface SimpleHlsAudioOnlyEngineState { userAudioTrackSelection?: Partial; /** * The CDNs the source is served from, in manifest priority order (mirrors - * HLS content steering's `PATHWAY-PRIORITY`). Owned by `resolveCdnPriority`, + * HLS content steering's `PATHWAY-PRIORITY`). Owned by `deriveCdnPriority`, * read by `track-switching`'s `preferActiveCdn` scope. Only meaningful for * redundant-stream sources; a single-CDN source has one entry. */ cdnPriority?: string[]; + /** + * CDN ids currently in failover cooldown — read by `track-switching`'s + * `excludeFailedCdns` constraint, which prunes their tracks so the active-CDN + * scope falls to the next CDN. Empty / absent means all CDNs are eligible. + */ + failedCdns?: string[]; currentTime?: number; loadActivated?: boolean; } @@ -94,14 +102,24 @@ export interface SimpleHlsAudioOnlyEngineConfig parsePresentation?: ParsePresentation; forwardBuffer?: Partial; backBuffer?: Partial; + /** Multi-CDN failover monitor tuning. Defaults: `DEFAULT_FAILOVER_MONITOR_CONFIG`. */ + failover?: Partial; + /** + * Derive a CDN grouping key from a track URL (used by `cdnPriority`, the + * failover trip, and the track-switching CDN rules — one function read by all). + * Defaults to the URL origin; override to key on e.g. Mux's `cdn=` param. + */ + getCdnId?: GetCdnId; } // ============================================================================ // Audio-Only HLS Playback Engine // ============================================================================ -// Materializes the consumer-input slot `userAudioTrackSelection` (only read by -// switchAudioTrack, produced by no behavior) in addition to forwarding refs. +// Materializes input slots no composed behavior produces — `userAudioTrackSelection` +// (switchAudioTrack only reads it) — in addition to forwarding refs. `failedCdns` +// is owned by `setupFailoverMonitor`, so it's already materialized and reachable +// on the `onSignalsReady` refs without being listed here. const shareSignals = makeShareSignals([ 'userAudioTrackSelection', ]); @@ -160,7 +178,12 @@ export function createHlsAudioOnlyEngine( // not load-bearing here today. It earns its place for forward-consistency // with the default engine and for future failover / steering, where the // active CDN changes dynamically (and selection stays reactive either way). - resolveCdnPriority, + deriveCdnPriority, + + // CDN failover cooldown: watches `failedCdns` (tripped directly by audio + // track resolution on a failed media-playlist fetch) and removes each CDN + // once its cooldown lapses. + setupFailoverMonitor, // Audio track selection — slot owner with filter reactivity. // Mid-stream flush on language switch is handled in segment-loader's diff --git a/packages/spf/src/playback/engines/hls/engine.ts b/packages/spf/src/playback/engines/hls/engine.ts index 71884ed1..874622a2 100644 --- a/packages/spf/src/playback/engines/hls/engine.ts +++ b/packages/spf/src/playback/engines/hls/engine.ts @@ -16,6 +16,7 @@ import { } from '../../../media/dom/text/text-track-slots'; import { parseMultivariantPlaylist } from '../../../media/hls/parse-multivariant'; import type { AudioTrack, MaybeResolvedPresentation, VideoTrack } from '../../../media/types'; +import type { GetCdnId } from '../../../media/utils/cdn'; import { getResolvedSelectedTrackDuration } from '../../../media/utils/track-selection'; import type { BandwidthConfig, BandwidthState } from '../../../network/bandwidth-estimator'; import type { SegmentLoaderActor } from '../../actors/dom/segment-loader'; @@ -26,6 +27,7 @@ import { calculatePresentationDuration, type PresentationDurationResolver, } from '../../behaviors/calculate-presentation-duration'; +import { deriveCdnPriority } from '../../behaviors/derive-cdn-priority'; import { endOfStream } from '../../behaviors/dom/end-of-stream'; import { loadAudioSegments, loadTextTrackSegments, loadVideoSegments } from '../../behaviors/dom/load-segments'; import { setupAudioBufferActors, setupVideoBufferActors } from '../../behaviors/dom/setup-buffer-actors'; @@ -35,10 +37,10 @@ import { syncTextTracks } from '../../behaviors/dom/sync-text-tracks'; import { trackCurrentTime } from '../../behaviors/dom/track-current-time'; import { trackLoadTriggers } from '../../behaviors/dom/track-load-triggers'; import { updateMediaSourceDuration } from '../../behaviors/dom/update-mediasource-duration'; -import { resolveCdnPriority } from '../../behaviors/resolve-cdn-priority'; import { type ParsePresentation, resolvePresentation } from '../../behaviors/resolve-presentation'; import { resolveAudioTrack, resolveTextTrack, resolveVideoTrack } from '../../behaviors/resolve-track'; import { selectTextTrack } from '../../behaviors/select-tracks'; +import { type FailoverMonitorConfig, setupFailoverMonitor } from '../../behaviors/setup-failover-monitor'; import { syncPreload } from '../../behaviors/sync-preload'; import { switchAudioTrack, switchVideoTrack } from '../../behaviors/track-switching'; @@ -75,13 +77,21 @@ export interface SimpleHlsEngineState { /** * The CDNs the source is served from (track-URL origins), in manifest * priority order — most-preferred first (mirrors HLS content steering's - * `PATHWAY-PRIORITY`). Owned by `resolveCdnPriority`, read by + * `PATHWAY-PRIORITY`). Owned by `deriveCdnPriority`, read by * `track-switching`'s `preferActiveCdn` scope, which narrows to the * highest-priority CDN with surviving tracks so video / audio / text stay on * one host. Only meaningful for redundant-stream sources; a single-CDN source * has one entry. */ cdnPriority?: string[]; + /** + * CDN ids (origins) currently in failover cooldown — written by the CDN + * monitor when a host fails too often, read by `track-switching`'s + * `excludeFailedCdns` hard constraint, which prunes their tracks so the + * active-CDN scope falls to the next CDN in `cdnPriority`. Empty / absent + * means all CDNs are eligible. + */ + failedCdns?: string[]; currentTime?: number; loadActivated?: boolean; } @@ -196,6 +206,21 @@ export interface SimpleHlsEngineConfig extends ShareSignalsConfig; + /** + * Multi-CDN failover monitor tuning. `cooldownMs` is how long a CDN stays + * excluded after a failed fetch trips it. Defaults: + * `DEFAULT_FAILOVER_MONITOR_CONFIG` (300s). Only meaningful for redundant-stream + * sources. + */ + failover?: Partial; + /** + * How to derive a CDN grouping key from a track URL — used to build + * `cdnPriority`, to record the failover trip in `failedCdns`, and by the + * track-switching CDN scope + failover constraint. One function, read by all of + * them, so the keys stay comparable. Defaults to the URL origin; override to + * key on something else (e.g. Mux's `cdn=` query param). + */ + getCdnId?: GetCdnId; } // ============================================================================ @@ -205,9 +230,11 @@ export interface SimpleHlsEngineConfig extends ShareSignalsConfig([ 'userVideoTrackSelection', @@ -273,7 +300,12 @@ export function createSimpleHlsEngine( // media-playlist fetch to the wrong CDN before correcting. Symmetric // redundant streams (the norm) never hit it — the first-listed CDN is // already the primary we'd pick anyway. - resolveCdnPriority, + deriveCdnPriority, + + // CDN failover cooldown: owns the expiry half of failover — watches + // `failedCdns` (tripped directly by track resolution on a failed + // media-playlist fetch) and removes each CDN once its cooldown lapses. + setupFailoverMonitor, // Track selection (reads config for initial preferences). // Video selection lives in switchVideoTrack (composed below); diff --git a/packages/spf/src/playback/engines/hls/tests/engine.test.ts b/packages/spf/src/playback/engines/hls/tests/engine.test.ts index 48254c6d..1abf0bf0 100644 --- a/packages/spf/src/playback/engines/hls/tests/engine.test.ts +++ b/packages/spf/src/playback/engines/hls/tests/engine.test.ts @@ -133,7 +133,7 @@ describe('createSimpleHlsEngine', () => { }); it('keeps audio on the same CDN as video even when the audio rendition order differs', async () => { - // Order-effect guard: `resolveCdnPriority` derives the list from track order, + // Order-effect guard: `deriveCdnPriority` derives the list from track order, // so a same-ordered source can't distinguish "scope applied" from "scope is // a no-op". This source is doubly adversarial to the desired result: the // audio selection set comes BEFORE video in the manifest, and within it the @@ -210,6 +210,176 @@ describe('createSimpleHlsEngine', () => { engine.destroy(); }); + it('fails over video and audio to the next CDN when one is marked failed', async () => { + const flush = () => Promise.resolve().then(() => Promise.resolve()); + const engine = createSimpleHlsEngine(); + + const videoTrack = (id: string, host: string): PartiallyResolvedVideoTrack => ({ + type: 'video', + id, + codecs: [], + url: `https://${host}/${id}.m3u8`, + bandwidth: 2_400_000, + mimeType: 'video/mp4', + }); + const audioTrack = (id: string, host: string): PartiallyResolvedAudioTrack => ({ + type: 'audio', + id, + codecs: ['mp4a.40.2'], + url: `https://${host}/${id}.m3u8`, + bandwidth: 128_000, + mimeType: 'audio/mp4', + groupId: 'audio', + name: id, + sampleRate: 48_000, + channels: 2, + }); + + engine.state.presentation.set({ + id: 'pres-failover', + url: 'https://cdn-a.example.com/master.m3u8', + startTime: 0, + selectionSets: [ + { + id: 'v', + type: 'video', + switchingSets: [ + { + id: 'vs', + type: 'video', + tracks: [videoTrack('vid-a', 'cdn-a.example.com'), videoTrack('vid-b', 'cdn-b.example.com')], + }, + ], + }, + { + id: 'a', + type: 'audio', + switchingSets: [ + { + id: 'as', + type: 'audio', + tracks: [audioTrack('aud-a', 'cdn-a.example.com'), audioTrack('aud-b', 'cdn-b.example.com')], + }, + ], + }, + ], + } as Presentation); + await flush(); + + // Primary CDN initially. + expect(engine.state.selectedVideoTrackId.get()).toBe('vid-a'); + expect(engine.state.selectedAudioTrackId.get()).toBe('aud-a'); + + // Mark cdn-a failed → both types fail over to cdn-b coherently. + engine.state.failedCdns.set(['https://cdn-a.example.com']); + await flush(); + expect(engine.state.selectedVideoTrackId.get()).toBe('vid-b'); + expect(engine.state.selectedAudioTrackId.get()).toBe('aud-b'); + + // cdn-a recovers → both return to the primary. + engine.state.failedCdns.set([]); + await flush(); + expect(engine.state.selectedVideoTrackId.get()).toBe('vid-a'); + expect(engine.state.selectedAudioTrackId.get()).toBe('aud-a'); + + engine.destroy(); + }); + + it('auto-fails-over when a CDN fetch fails (monitor trips, failedCdns set)', async () => { + const engine = createSimpleHlsEngine({ failover: { cooldownMs: 60_000 } }); + + // cdn-a is down (media-playlist fetch rejects); cdn-b serves a valid playlist. + globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : String((input as Request).url ?? input); + if (url.includes('cdn-a')) throw new TypeError('cdn-a unreachable'); + return new Response('#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXTINF:10.0,\nseg-1.m4s\n#EXT-X-ENDLIST'); + }) as typeof fetch; + + const videoTrack = (id: string, host: string): PartiallyResolvedVideoTrack => ({ + type: 'video', + id, + codecs: [], + url: `https://${host}/${id}.m3u8`, + bandwidth: 2_400_000, + mimeType: 'video/mp4', + }); + + engine.state.presentation.set({ + id: 'pres-failover', + url: 'https://cdn-a.example.com/master.m3u8', + startTime: 0, + selectionSets: [ + { + id: 'v', + type: 'video', + switchingSets: [ + { + id: 'vs', + type: 'video', + tracks: [videoTrack('vid-a', 'cdn-a.example.com'), videoTrack('vid-b', 'cdn-b.example.com')], + }, + ], + }, + ], + } as Presentation); + + // The primary (cdn-a) is picked first, its media-playlist fetch fails, the + // monitor trips it, the constraint prunes it, and the scope fails over to + // cdn-b — all without any external failedCdns write. + await vi.waitFor(() => { + expect(engine.state.failedCdns.get()).toEqual(['https://cdn-a.example.com']); + expect(engine.state.selectedVideoTrackId.get()).toBe('vid-b'); + }); + + engine.destroy(); + }); + + it('honors a custom getCdnId across cdnPriority, the trip, and the constraint/scope', async () => { + // Key CDNs on the `cdn=` query param instead of origin. Both variants share a + // host, so origin-based identity would see ONE CDN (no redundancy); the + // custom resolver must be respected at every site for failover to work. + const getCdnId = (url: string) => new URL(url).searchParams.get('cdn') ?? url; + const engine = createSimpleHlsEngine({ getCdnId, failover: { cooldownMs: 60_000 } }); + + globalThis.fetch = vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : String((input as Request).url ?? input); + if (url.includes('cdn=a')) throw new TypeError('cdn-a unreachable'); + return new Response('#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXTINF:10.0,\nseg-1.m4s\n#EXT-X-ENDLIST'); + }) as typeof fetch; + + const videoTrack = (id: string, cdn: string): PartiallyResolvedVideoTrack => ({ + type: 'video', + id, + codecs: [], + url: `https://cdn.example.com/${id}.m3u8?cdn=${cdn}`, + bandwidth: 2_400_000, + mimeType: 'video/mp4', + }); + + engine.state.presentation.set({ + id: 'pres-custom-cdn', + url: 'https://cdn.example.com/master.m3u8', + startTime: 0, + selectionSets: [ + { + id: 'v', + type: 'video', + switchingSets: [{ id: 'vs', type: 'video', tracks: [videoTrack('vid-a', 'a'), videoTrack('vid-b', 'b')] }], + }, + ], + } as Presentation); + + await vi.waitFor(() => { + // deriveCdnPriority keyed on the param (not origin → not a single CDN). + expect(engine.state.cdnPriority.get()).toEqual(['a', 'b']); + // The trip recorded the param key, and the constraint + scope failed over. + expect(engine.state.failedCdns.get()).toEqual(['a']); + expect(engine.state.selectedVideoTrackId.get()).toBe('vid-b'); + }); + + engine.destroy(); + }); + it('allows patching state and owners from outside', async () => { const engine = createSimpleHlsEngine(); diff --git a/packages/spf/src/playback/engines/hls/tests/failover-smoke.test.ts b/packages/spf/src/playback/engines/hls/tests/failover-smoke.test.ts new file mode 100644 index 00000000..2b2e8145 --- /dev/null +++ b/packages/spf/src/playback/engines/hls/tests/failover-smoke.test.ts @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { isResolvedTrack, type MaybeResolvedPresentation } from '../../../../media/types'; +import { createSimpleHlsEngine } from '../engine'; + +// Live smoke test for multi-CDN failover against a real Mux `redundant_streams` +// source. It hits the network (the manifest + the surviving CDN are fetched for +// real), so it's gated behind VITE_FAILOVER_SMOKE and skipped in the default run. +// +// VITE_FAILOVER_SMOKE=1 pnpm -F @videojs/spf test src/playback/engines/hls/tests/failover-smoke.test.ts +// +// We can't make a real Mux CDN drop requests, so the failure is "hacked": a +// fetch wrapper rejects every request to the primary origin while letting the +// manifest and the backup origin hit the real network. +const SMOKE = (import.meta as unknown as { env?: Record }).env?.VITE_FAILOVER_SMOKE; + +const REDUNDANT_URL = 'https://stream.mux.com/s41JYeqIpBMBzE4OzxDyGR2yrp2hD1CQ6gJN9SlVGDQ.m3u8?redundant_streams=true'; + +// This asset duplicates every variant across two origins; edgemv is listed +// first, so it resolves to cdnPriority[0] and is the one we block. +const PRIMARY = 'edgemv.mux.com'; +const BACKUP = 'fastly.mux.com'; + +const hostOf = (url: string): string => new URL(url).host; + +function selectedVideoTrack(presentation: MaybeResolvedPresentation | undefined, id: string | undefined) { + if (!presentation || !id) return undefined; + for (const set of presentation.selectionSets ?? []) { + for (const sw of set.switchingSets) { + const track = sw.tracks.find((t) => t.id === id); + if (track) return track; + } + } + return undefined; +} + +describe.skipIf(!SMOKE)('multi-CDN failover (live smoke)', () => { + let realFetch: typeof globalThis.fetch; + + afterEach(() => { + globalThis.fetch = realFetch; + }); + + it('fails over to the backup CDN when the primary is unreachable, then recovers', async () => { + realFetch = globalThis.fetch; + let blockPrimary = true; + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.href : (input as Request).url; + if (blockPrimary && url.includes(PRIMARY)) return Promise.reject(new TypeError('blocked (smoke)')); + return realFetch(input as RequestInfo, init); + }) as typeof fetch; + + const engine = createSimpleHlsEngine({ failover: { cooldownMs: 4000 } }); + engine.state.presentation.set({ url: REDUNDANT_URL } as MaybeResolvedPresentation); + + // The primary is picked first, its media-playlist fetch fails, the trip + // lands in failedCdns, the constraint prunes it, and the selected video + // track resolves on the backup CDN. + await vi.waitFor( + () => { + expect(engine.state.cdnPriority.get()?.length).toBe(2); + expect(engine.state.failedCdns.get()?.some((cdn) => cdn.includes(PRIMARY))).toBe(true); + const track = selectedVideoTrack(engine.state.presentation.get(), engine.state.selectedVideoTrackId.get()); + expect(track).toBeDefined(); + expect(hostOf(track!.url)).toContain(BACKUP); + expect(isResolvedTrack(track!)).toBe(true); + }, + { timeout: 20_000, interval: 250 } + ); + + // Recovery: unblock the primary. Once its cooldown lapses it leaves + // failedCdns and (being cdnPriority[0]) is preferred again — selection flips + // back and the primary playlist now resolves for real. + blockPrimary = false; + await vi.waitFor( + () => { + expect(engine.state.failedCdns.get()?.some((cdn) => cdn.includes(PRIMARY))).toBe(false); + const track = selectedVideoTrack(engine.state.presentation.get(), engine.state.selectedVideoTrackId.get()); + expect(track).toBeDefined(); + expect(hostOf(track!.url)).toContain(PRIMARY); + expect(isResolvedTrack(track!)).toBe(true); + }, + { timeout: 20_000, interval: 250 } + ); + + await engine.destroy(); + }, 60_000); +}); diff --git a/packages/spf/src/playback/primitives/failover-fetch.ts b/packages/spf/src/playback/primitives/failover-fetch.ts new file mode 100644 index 00000000..15255175 --- /dev/null +++ b/packages/spf/src/playback/primitives/failover-fetch.ts @@ -0,0 +1,63 @@ +import { type ReadonlySignal, type Signal, update } from '../../core/signals/primitives'; +import type { MaybeResolvedPresentation } from '../../media/types'; +import { addFailedCdn, getCdnId as defaultGetCdnId, type GetCdnId } from '../../media/utils/cdn'; +import { findTrackById } from '../../media/utils/tracks'; +import type { FetchOptions, Resource } from '../../network/fetch'; + +type SelectedTrackKey = 'selectedVideoTrackId' | 'selectedAudioTrackId' | 'selectedTextTrackId'; + +/** + * State a failover-decorated fetch reads: the presentation, the per-type + * selected-track slot, and the failover monitor's `failedCdns`. + * + * `failedCdns` is *optional* — the failover monitor owns that slot, so a + * behavior's narrow state is assignable here without the behavior declaring it + * (and the intersection shares keys, so it isn't a weak type). When no monitor + * is composed the slot is absent and tracking no-ops. + */ +type FailoverState = { + presentation: ReadonlySignal; + failedCdns?: Signal; +} & { [P in K]: ReadonlySignal }; + +/** Any `Resource`-addressable fetch — both `FetchText` and `FetchBytes` qualify. */ +type FailoverableFetch = (addressable: Resource, options?: FetchOptions) => Promise; + +/** + * Decorate a fetch so a failed request trips the **selected track's** CDN into + * `failedCdns`. The decorated fetch's type is preserved, so this wraps both + * `resolve-track`'s playlist `FetchText` and the segment loaders' `FetchBytes`. + * + * The CDN id comes from the selected track's media-playlist URL, never the + * failed addressable: a segment URL resolves relative to its playlist and, per + * RFC 3986, drops the playlist's query string (`…/r.m3u8?cdn=fastly` → `…/0.ts`), + * so a query-keyed `getCdnId` (e.g. Mux's `cdn=`) keyed on it would derive an id + * that never matches the ones `deriveCdnPriority` / track-switching build from + * `track.url`. The in-flight fetch belongs to the selected track — a source or + * track switch aborts it, and aborts don't trip — so the selected track is the + * right CDN to fail over. For `resolve-track` the resolving track *is* the + * selected track, so this is identical to keying on its addressable. + * + * No-op when no failover monitor is composed (it owns the signal) or the + * selected track can't be located. + */ +export function failoverFetch( + baseFetch: Fetch, + state: FailoverState, + config: { selectedKey: K; getCdnId?: GetCdnId } +): Fetch { + const getCdnId = config.getCdnId ?? defaultGetCdnId; + return (async (addressable: Resource, options?: FetchOptions) => { + try { + return await baseFetch(addressable, options); + } catch (error) { + if (!options?.signal?.aborted && state.failedCdns) { + const presentation = state.presentation.get(); + const trackId = state[config.selectedKey].get(); + const track = presentation && trackId ? findTrackById(presentation, trackId) : undefined; + if (track) update(state.failedCdns, (cdns) => addFailedCdn(cdns, getCdnId(track.url))); + } + throw error; + } + }) as Fetch; +} diff --git a/packages/spf/src/playback/primitives/tests/failover-fetch.test.ts b/packages/spf/src/playback/primitives/tests/failover-fetch.test.ts new file mode 100644 index 00000000..43e933db --- /dev/null +++ b/packages/spf/src/playback/primitives/tests/failover-fetch.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; +import { signal } from '../../../core/signals/primitives'; +import type { MaybeResolvedPresentation } from '../../../media/types'; +import type { FetchText } from '../../../network/fetch'; +import { failoverFetch } from '../failover-fetch'; + +const presentationWithVideo = (url: string): MaybeResolvedPresentation => ({ + url: 'https://cdn-a.example.com/master.m3u8', + selectionSets: [ + { + id: 'video-set', + type: 'video', + switchingSets: [{ id: 'sw', type: 'video', tracks: [{ id: 'v0', type: 'video', url, bandwidth: 0 }] }], + }, + ] as MaybeResolvedPresentation['selectionSets'], +}); + +// A query-keyed getCdnId (e.g. Mux's `cdn=`), falling back to origin when the +// param is absent — as it is on segment URLs after relative resolution. +const byCdnParam = (url: string) => new URL(url).searchParams.get('cdn') ?? new URL(url).origin; + +const makeState = (presentation: MaybeResolvedPresentation, selectedId: string | undefined) => ({ + presentation: signal(presentation), + selectedVideoTrackId: signal(selectedId), + failedCdns: signal(undefined), +}); + +const reject: FetchText = async () => { + throw new Error('boom'); +}; + +// The segment URL has dropped the playlist's `?cdn=` param during relative +// resolution, so it's not a valid CDN-identity source. +const segment = { url: 'https://cdn-a.example.com/0.ts' }; + +describe('failoverFetch', () => { + it('trips the selected track CDN — not the failed addressable — on a failed fetch', async () => { + const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8?cdn=fastly'), 'v0'); + const fetch = failoverFetch(reject, state, { selectedKey: 'selectedVideoTrackId', getCdnId: byCdnParam }); + + await expect(fetch(segment)).rejects.toThrow('boom'); + // Keyed on the track URL (`cdn=fastly`), not the param-less segment URL. + expect(state.failedCdns.get()).toEqual(['fastly']); + expect(byCdnParam(segment.url)).not.toBe('fastly'); + }); + + it('does not trip on an aborted fetch', async () => { + const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8?cdn=fastly'), 'v0'); + const fetch = failoverFetch(reject, state, { selectedKey: 'selectedVideoTrackId', getCdnId: byCdnParam }); + const controller = new AbortController(); + controller.abort(); + + await expect(fetch(segment, { signal: controller.signal })).rejects.toThrow(); + expect(state.failedCdns.get()).toBeUndefined(); + }); + + it('no-ops when the selected track cannot be located', async () => { + const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8?cdn=fastly'), 'missing'); + const fetch = failoverFetch(reject, state, { selectedKey: 'selectedVideoTrackId', getCdnId: byCdnParam }); + + await expect(fetch(segment)).rejects.toThrow(); + expect(state.failedCdns.get()).toBeUndefined(); + }); + + it('passes a successful fetch through unchanged', async () => { + const state = makeState(presentationWithVideo('https://cdn-a.example.com/r.m3u8'), 'v0'); + const ok: FetchText = async () => 'body'; + const fetch = failoverFetch(ok, state, { selectedKey: 'selectedVideoTrackId' }); + + await expect(fetch(segment)).resolves.toBe('body'); + expect(state.failedCdns.get()).toBeUndefined(); + }); +}); diff --git a/packages/spf/src/playback/behaviors/track-types.ts b/packages/spf/src/playback/primitives/track-types.ts similarity index 100% rename from packages/spf/src/playback/behaviors/track-types.ts rename to packages/spf/src/playback/primitives/track-types.ts diff --git a/packages/spf/src/playback/primitives/tsconfig.json b/packages/spf/src/playback/primitives/tsconfig.json new file mode 100644 index 00000000..17dc9141 --- /dev/null +++ b/packages/spf/src/playback/primitives/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "lib": ["ES2022", "WebWorker"], + "exactOptionalPropertyTypes": false, + "declarationDir": "../../../types/playback/primitives" + }, + "references": [ + { "path": "../../../../utils" }, + { "path": "../../core" }, + { "path": "../../network" }, + { "path": "../../media" } + ], + "include": ["./*.ts", "./tests/**/*.ts"] +} diff --git a/packages/spf/vitest.config.ts b/packages/spf/vitest.config.ts index b9f8b2b6..8e66eff4 100644 --- a/packages/spf/vitest.config.ts +++ b/packages/spf/vitest.config.ts @@ -42,7 +42,11 @@ export default defineConfig({ extends: true, test: { name: 'behaviors', - include: ['src/playback/behaviors/**/*.test.ts', 'src/playback/actors/**/*.test.ts'], + include: [ + 'src/playback/behaviors/**/*.test.ts', + 'src/playback/actors/**/*.test.ts', + 'src/playback/primitives/**/*.test.ts', + ], exclude: ['src/playback/behaviors/dom/**', 'src/playback/actors/dom/**'], }, }, diff --git a/tsconfig.json b/tsconfig.json index e860ba15..ce28dea6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -23,6 +23,7 @@ { "path": "packages/spf/src/network" }, { "path": "packages/spf/src/media" }, { "path": "packages/spf/src/media/dom" }, + { "path": "packages/spf/src/playback/primitives" }, { "path": "packages/spf/src/playback/behaviors" }, { "path": "packages/spf/src/playback/behaviors/dom" }, { "path": "packages/spf/src/playback/actors" },