From 057f32573e5a5178e1508dd21fa5f98f05a4eb2a Mon Sep 17 00:00:00 2001 From: Christian Pillsbury Date: Thu, 4 Jun 2026 09:53:13 -0700 Subject: [PATCH] feat(spf): multi-track audio + skills building features and behaviors (#1605) Co-authored-by: Claude Opus 4.7 (1M context) --- .claude/skills/spf-implement-feature/SKILL.md | 591 +++++++++++++++--- apps/sandbox/app/shared/sources.ts | 6 + .../templates/spf-segment-loading/index.html | 33 +- .../templates/spf-segment-loading/main.ts | 162 ++++- internal/design/spf/features/audio-abr.md | 58 +- .../design/spf/features/audio-playback.md | 56 +- internal/design/spf/features/clusters.md | 20 + .../spf/features/multi-language-audio.md | 178 ++++-- .../spf/use-cases/audio-only-mode-override.md | 27 +- packages/spf/src/all.ts | 6 +- .../core/composition/create-composition.ts | 2 +- .../spf/src/media/abr/quality-selection.ts | 32 +- .../media/abr/tests/quality-selection.test.ts | 46 +- .../spf/src/media/primitives/select-tracks.ts | 66 +- .../primitives/tests/select-tracks.test.ts | 21 +- .../src/playback/actors/dom/segment-loader.ts | 90 ++- .../src/playback/actors/dom/source-buffer.ts | 22 +- .../actors/dom/tests/segment-loader.test.ts | 238 +++++++ .../actors/dom/tests/source-buffer.test.ts | 31 + .../calculate-presentation-duration.ts | 2 +- .../behaviors/dom/setup-buffer-actors.ts | 9 +- .../behaviors/dom/sync-text-tracks.ts | 2 +- .../playback/behaviors/quality-switching.ts | 342 ---------- .../src/playback/behaviors/select-tracks.ts | 100 ++- .../behaviors/tests/select-tracks.test.ts | 91 ++- ...tching.test.ts => track-switching.test.ts} | 268 +++++++- .../src/playback/behaviors/track-switching.ts | 438 +++++++++++++ .../playback/engines/hls/engine-audio-only.ts | 20 +- .../spf/src/playback/engines/hls/engine.ts | 27 +- .../hls/tests/engine-audio-only.test.ts | 15 + 30 files changed, 2257 insertions(+), 742 deletions(-) create mode 100644 packages/spf/src/playback/actors/dom/tests/segment-loader.test.ts delete mode 100644 packages/spf/src/playback/behaviors/quality-switching.ts rename packages/spf/src/playback/behaviors/tests/{quality-switching.test.ts => track-switching.test.ts} (63%) create mode 100644 packages/spf/src/playback/behaviors/track-switching.ts diff --git a/.claude/skills/spf-implement-feature/SKILL.md b/.claude/skills/spf-implement-feature/SKILL.md index 6ec67a7e..a7b271ca 100644 --- a/.claude/skills/spf-implement-feature/SKILL.md +++ b/.claude/skills/spf-implement-feature/SKILL.md @@ -25,11 +25,12 @@ that doesn't match the doc's grounding, drifts beyond the agreed phase scope, misses the conventions catalog, or skips the doc-update cascade that keeps the feature registry current as code lands. -Steps 1–2 are the load-bearing setup. Skipping them produces implementations +Steps 1–3 are the load-bearing setup. Skipping them produces implementations that look superficially correct but anchor on the wrong scope, miss the doc's -open questions, or fail to coordinate with cross-cutting concerns. Steps 3–7 -only make sense once the feature, sources, and intent are named. Step 8 (doc -update as living artifact) is where the registry stays in sync with code. +open questions, miss adjacent features whose shape constrains this one, or +fail to coordinate with cross-cutting concerns. Steps 4–8 only make sense +once the feature, composition target, fold-ins, and intent are named. Step 9 +(doc update as living artifact) is where the registry stays in sync with code. ## Usage @@ -68,7 +69,7 @@ Discipline: - **Every doc revision is explicit and surfaced** to the user during the implementation pass. Never silent. -- **Step 8 consolidates** the cumulative doc update reflecting all +- **Step 9 consolidates** the cumulative doc update reflecting all revisions made during implementation — it's not the *only* revision point, just the cumulative one. - **Open questions are markers**, not absent specs — the implementation @@ -136,6 +137,158 @@ Downstream skills routed-to: have meant another (especially: a request that's actually a use case routed-here as a feature, or vice versa). Always surface and confirm. +- **Standard-vs-use-case composition routing skipped** — Step 1 must + confirm whether the implementation effort is for the base/standard + composition or for a specific use-case composition. Default is + base, but use-case-FOR implementations have different framing: the + use-case doc supplies the destination-architecture sketch; cluster + traversal narrows. Worked example: implementing + `multi-language-audio` for the base engine forces the cluster's + destination-architecture frame; implementing it as part of + `audio-only-mode-override` Phase 2 narrows to the use case's + variant engine. + +- **Destination architecture unframed** — for features in the base + composition, scoping without sketching the cluster's destination + architecture produces local-optimal cuts that don't fit the larger + shape. The destination sketch is Step 2's reference frame for the + fold-in assessment; without it, the assessment defaults to "is + this candidate adjacent to my feature?" rather than "does my + feature need to align with where this cluster is heading?" + +- **Doc-related-features-walked-without-cluster-traversal** — + relying solely on the feature doc's flat *Related features* list + misses architectural-axis siblings reachable via `clusters.md` + (primary cluster siblings + cross-cluster axis siblings). The + *Related features* list is a starting point; the cluster traversal + is what surfaces the architectural axis. Worked example: + `multi-language-audio`'s *Related features* list includes + `capability-probing` as a "candidate," but doesn't loudly surface + that capability-probing sits on the same + "filtering/prioritizing/selecting tracks of a given type" axis — + cluster traversal makes that axis explicit and the fold-in + candidacy clearer. + +- **Cross-cluster axis traversal short-circuit** — treating the + feature doc's *Related features* list as the *ceiling* for + cross-cluster candidates rather than the *floor*. Has two faces: + (i) **surfacing failure** — not enumerating the cross-cluster + cousins at all; (ii) **assessment failure** — enumerating them but + rating them all "ignore for now," skipping past the + design-with-in-mind landing zone. + + For cluster-C (track & variant registry) features, at minimum + check capability-probing (D), rendition-selection-caps (E), + multi-cdn-failover (G), drm-support (H) — these cousins all + participate in the "filter / prioritize / select track candidates" + axis (see `clusters.md` → *Selection / filtering across clusters*). + Analogous cross-cluster cousins exist for other primary clusters; + consult the clusters doc before defaulting to "no cross-cluster + candidates." + + **DWIM is the default recommendation for cross-cluster cousins**, + unless Impact-if-deferred pushes higher or Shape-constraint-if- + deferred is genuinely low. The clusters doc names this + explicitly: *"Candidates often land as design-with-in-mind rather + than full fold-in (each cluster owns its own primitives), but + surfacing them keeps the cluster-C feature's shape from painting + into a corner."* See Step 2d's *Default-recommended outcome per + candidate type* table — cross-cluster cousins are the row where + DWIM is the prior, not Ignore. + + Worked example: `multi-language-audio`'s *Related features* lists + `capability-probing` but omits `rendition-selection-caps`, + `multi-cdn-failover`, `drm-support`. The implementation pass + should rate all four as DWIM: codec filtering composes with the + `userAudioTrackSelection` filter slot on the same axis; audio + caps would bias the same candidate set the filter narrows; + per-language URI rotation needs to stay extensible during + mid-stream-switch design; per-language key-system filtering + interacts with selection. Rating any of these "ignore" without + surfacing as DWIM is the assessment-failure shape; rating all + four as DWIM without surfacing each via `AskUserQuestion` is the + narrative-batched-skip shape (see next entry). + +- **Narrative-batched-skip of `AskUserQuestion` for "obvious" + candidates** — collapsing multiple fold-in candidates into a single + narrative sentence ("the remaining cross-cluster cousins all + default to ignore for now") and skipping the per-candidate + `AskUserQuestion` because the recommendation feels obvious. The + structured presentation is the load-bearing pressure that surfaces + assessment misjudgments — bypassing it lets misjudged + recommendations slip through unchallenged. This failure mode often + pairs with *Cross-cluster axis traversal short-circuit*'s + assessment-failure face: candidates rated "ignore for now" feel + uncontroversial enough to batch, and the batching is what hides + the misjudgment. + + **Discipline.** Use `AskUserQuestion` per candidate **regardless of + recommendation strength**, even when the recommendation feels + obvious. If multiple candidates would otherwise be batched into one + narrative paragraph, group by shared rationale into 2–3 + `AskUserQuestion` calls (4 candidates per call, per the + `AskUserQuestion` cap) — but every candidate appears as a discrete + row with discrete options, with DWIM and Ignore both visible in the + option list for cross-cluster cousins specifically. + + Worked example: implementation pass enumerated 4 cross-cluster + cousins (cap-probing / caps / CDN / DRM) and assessed all as + "ignore for now," then narrative-batched ("the cross-cluster cousins + all default to ignore"). User pushback identified 3 of 4 as + design-with-in-mind that the narrative-batch silently suppressed. + Per-candidate `AskUserQuestion` with DWIM visible would have caught + the misjudgment by the first candidate. + +- **Order-inversion not surfaced** — when this feature anchors on a + sibling's slot-owner shape (a *destination-architecture sibling*), + implementing the sibling first may be the right route. This is a + legitimate fold-in outcome distinct from full / partial / + design-with-in-mind / ignore — surface it explicitly to the user, + don't silently reject as "we're implementing this feature, not + that one." Signal: a sibling at `definition: technical` (or deeper) + carries the destination architecture in its *Phases of complexity* + table. Worked example: `multi-language-audio` anchors on + `switchAudioQuality` (audio-abr Phase 3 + Phase 4 + + `selectedAudioTrackId` triple-writer characterization in Phase 5). + Implementing `multi-language-audio` while extending the current + `selectAudioTrack` shape — instead of introducing + `switchAudioQuality` per audio-abr's destination — would unwind + when audio-abr ships. The "audio-abr first" ordering must be + surfaced as an option before the fold-in walk, not buried inside + one candidate's per-question outcomes. + +- **Boil-the-ocean false positive** — rejecting fold-in candidates + reflexively as "too big" without considering partial-implementation. + The right question is "is there a focused partial-implementation + cut?" not "is the full feature big?" Worked example: rejecting + `audio-abr` as boil-the-ocean when a basic `switchAudioQuality` + shape that establishes placement + slot pattern is a focused + partial that gets audio-ABR's foundation in place without + implementing the full feature. + +- **Conflating partial-implementation with design-with-in-mind** — + these are distinct fold-in outcomes. Partial = concrete code lands; + design-with-in-mind = no new code, but this feature's shape is + constrained by the candidate's eventual needs. Treating them as + one outcome loses the distinction and produces either too little + code (skipping high-impact partials) or too much (writing + speculative code for design considerations). Worked example: + `audio-abr` is partial-implementation (basic + `switchAudioQuality`); `5.1-surround-selection` is + design-with-in-mind (no codec-change code, but mid-stream flush + orchestration left extensible). + +- **Value-curve check skipped** — scoping to "spec-compliant + baseline" without checking whether the cut produces + user-meaningful capability. Per `clusters.md`'s "can-play vs actual + support" distinction, a cut that ships only passive recognition / + spec-compliance is a value-curve red flag — either fold in more, + or be explicit with the user about the constraint. Worked example: + `multi-language-audio` Tier 1 alone produces "the engine + recognizes audio tracks and picks one at load time" — still only + "can-play" at the cluster's value-curve, since the consumer can't + dynamically select among them. + - **Treating feature doc as hardened spec** — refusing to revise the doc when implementation reveals new questions, refines framing, or surfaces drift. Inverse failure of silent-override (see @@ -192,13 +345,13 @@ Downstream skills routed-to: - **Status / Implementation-surface update skipped** — feature doc transitions from coarse → sketched as code lands. The doc must be updated as part of the implementation; deferring means future agents - see stale status. Step 8 enforces. + see stale status. Step 9 enforces. - **Downstream skill missing — silent inline implementation** — when a chunk hits a "Yes" row in the downstream-skill-needed table (new behavior, non-trivial behavior update), and the downstream skill doesn't exist yet, the failure mode is to silently apply discipline - ad-hoc. Step 6 explicitly surfaces this: branch on (i) defer chunk + ad-hoc. Step 7 explicitly surfaces this: branch on (i) defer chunk pending downstream skill, (ii) build the downstream skill inline now, (iii) apply discipline ad-hoc with explicit "extract later" flag. @@ -244,7 +397,28 @@ boundary: `/spf-document-use-case` first, then `/spf-implement-use-case`. - **Ambiguous between options** → surface to user; do not pick silently. -**1d. Gather sources.** (Once routing is confirmed and we're staying here.) +**1d. Confirm composition target.** (Once routing is confirmed.) + +The implementation effort's composition target shapes scope considerably. +Default is **standard composition** (`createSimpleHlsEngine`); use-case +compositions (`createHlsAudioOnlyEngine`, etc.) follow a narrower frame. + +If not explicit in the invocation, surface and confirm. Three answers: + +- **Standard composition** *(default)* — feature lands in + `createSimpleHlsEngine` (or its constituents). Step 2's + destination-architecture frame applies in full. +- **Use-case composition: ``** — feature lands in the + use case's variant engine. Step 2's destination-architecture frame + is supplied by the use-case doc; fold-in traversal narrows. +- **Both** — feature lands in standard composition; one or more + use-case compositions may need refactoring to absorb it. Treat + each use-case refactor as a fold-in candidate in Step 2. + +If the user didn't specify, default to **standard composition** and +flag the assumption in the Step 1 report so the user can correct. + +**1e. Gather sources.** (Once routing and composition target are confirmed.) Triangulate from every available source: @@ -270,46 +444,286 @@ Triangulate from every available source: **Stop and report back to the user** with: -1. The feature name and the agreed phase scope (Phase 1? subset of - Phase 1's rows? multiple phases?). -2. Sources consulted (with links). -3. Feature doc status — what's already there vs what needs implementing. -4. Likely template implementations (similar behaviors / actors / helpers). -5. Open questions blocking implementation (going into Step 2's - discussion). -6. **Recommended phase scope** for this implementation pass. +1. The feature name. +2. The confirmed composition target (standard / use-case / both). +3. Sources consulted (with links). +4. Feature doc status — what's already there vs what needs implementing. +5. Likely template implementations (similar behaviors / actors / helpers). +6. The feature's phases (from the doc) — surfaced for context, with + the phase-scope decision deferred to Step 3 once fold-in + assessment in Step 2 has shaped it. -### Step 2 — Discuss to resolve open questions +### Step 2 — Frame the work: destination architecture + fold-in assessment -An **explicit conversational stage** — not optional, not implicit. After -Step 1's report: +Bridges "this feature, alone" and "this feature, as one slice of a +larger architectural concern." Without it, scope-narrow choices that +look reasonable in isolation produce implementations that don't fit +the destination architecture, get reworked when adjacent features +land, or ship capabilities of low user-meaningful value. -- **Walk through the feature doc's Open questions section.** Which need - to resolve *before* coding? Which can stay open as known-unknowns? -- **Confirm phase scope.** Phase 1 only? Phase 1 + Phase 2 subset? - Specific phase rows? The implementation must scope to a concrete chunk - list. -- **Confirm composition mechanism per chunk** — (i) subtractive (no new - code), (ii) config-driven (existing behavior gains a knob), (iii) - new behavior (route to `/spf-create-behavior`), (iv) behavior update - with purpose change (route to `/spf-update-behavior`), (v) behavior - refactor with preserved purpose (route to `/refactor-behavior`), (vi) - media-layer / network-layer change (handle inline or defer per the +Five movements (2a–2e). All five are required for features in the +standard composition; for features being implemented FOR a specific +use-case composition (per Step 1d), 2b lightens considerably — the +use-case doc supplies the destination-architecture frame. + +**2a. Identify the architectural axis / cluster.** + +Per `features/clusters.md`, the feature lives in one or more clusters +and may also sit on a cross-cluster axis (e.g., "selecting / +filtering / prioritizing tracks of a given type"). Both matter: + +- **Primary cluster** — the section in `clusters.md` the feature + belongs to. Gives the direct siblings. +- **Cross-cluster axis** — abstractions shared with features in + other clusters. The clusters doc's *Cross-cluster patterns* + section captures some of these (multi-writer state slots, + constraint + filter, per-type specialization, + sampling-baked-into-loading). The axis the feature sits on is + often inferable from the feature's verb-or-noun-phrase: "select" / + "filter" / "prioritize" / "sample" / "load" / "recover" / + "track-selection-for-type-X." + +Surface both: "this feature is in cluster X; it sits on the +[axis-name] axis alongside features in clusters X, Y, Z." + +**2b. Sketch the destination architecture.** + +For features being implemented for the **standard composition** (the +default), sketch where the cluster's destination architecture is +heading — the larger shape this feature is one slice of. The sketch +is the reference frame for the fold-in assessment. + +Format: a few sentences naming (i) the shape (multi-writer slot, +constraint + filter, per-type sibling triad, etc.), (ii) the moving +pieces (slots, behaviors, primitives), (iii) the open questions +about where things live (the cluster's known unknowns). The sketch +is necessarily approximate — its job is to give planning the right +frame, not commit to design. + +**Mine sibling docs for already-articulated destination shapes.** +Before sketching from scratch, scan cluster siblings' feature docs +for an already-articulated destination shape. A sibling at +`definition: technical` (or deeper) often carries the destination +architecture in its *Phases of complexity* table — behavior names, +slot ownership, multi-writer characterization. If found, the sketch +becomes a cross-reference to that sibling's articulated shape rather +than a fresh draft. When the destination sibling is itself +unimplemented (e.g., `audio-abr`'s `switchAudioQuality` for +`multi-language-audio`), this triggers an **ordering question** — +surface it to the user in Step 2's report, before the per-candidate +fold-in walk, per the *Order-inversion not surfaced* failure mode. + +For features being implemented **FOR a use-case composition**, the +use-case doc supplies most of this frame; cross-reference rather +than re-sketch. + +**2c. Enumerate fold-in candidates.** + +Walk three sources of candidates: + +- **Cluster siblings** (from 2a's primary cluster). +- **Cross-cluster axis siblings** (from 2a's axis — features in + other clusters that share the abstraction). For cluster-C + (registry) features, at minimum check capability-probing (D), + rendition-selection-caps (E), multi-cdn-failover (G), drm-support + (H) regardless of whether the feature doc lists them — see + `clusters.md` → *Selection / filtering across clusters*. Analogous + cross-cluster cousins exist for other primary clusters; consult + that section before defaulting to "no cross-cluster candidates." +- **The feature doc's *Related features* list** (catches anything + the cluster traversal missed — but treat the list as the *floor*, + not the ceiling). + +Candidates can be (i) whole features, (ii) specific phases of +features, or (iii) use-case-composition refactors (a minor refactor +to a use case's variant engine to absorb this feature's new +behaviors counts as a fold-in candidate). + +**2d. Assess each candidate; recommend an outcome.** + +Four criteria per candidate. **Shape-constraint-if-deferred comes +first** because it's the criterion that catches *design-with-in-mind* +— the other three are oriented toward "should we write code for this +candidate now?" and miss the design-shape question if it's not asked +separately. Per the *Cross-cluster axis traversal short-circuit* +failure mode. + +1. **Shape-constraint-if-deferred** — if we don't design this + feature with the candidate's eventual shape in mind, will the + structures we land make later integration awkward? Will slots, + composition seams, layering decisions, or filter-pipeline order + need to mutate when the candidate ships? High shape-constraint → + design-with-in-mind (or stronger); low shape-constraint → ignore + is safe. +2. **Impact if deferred** — would solving this feature without + considering the candidate produce a solution that needs + significant refactor / rework / re-characterization when the + candidate later lands? Or that would be insufficient when the + candidate lands? +3. **Overall speed if combined** — would partially or fully + implementing the candidate now produce overall less work than + serial implementation, even at moderate cost to this feature's + specific velocity? +4. **Scope discipline** — is the candidate's bundled work focused + enough to remain a discrete partial-implementation, or does it + sprawl into ocean-boiling territory? + +The fourth criterion is where the **boiling-the-ocean check** lives +— but it's a check on *scope sprawl*, not a default veto. Per the +*Boil-the-ocean false positive* failure mode, rejecting candidates +reflexively as "too big" misses the real question: "can this be cut +to a focused partial-implementation?" + +**Why four criteria, not three.** The previous three-criteria rubric +biased the recommendation toward "implement now or ignore" — when the +implement-now threshold failed, the natural cognitive landing slid +past design-with-in-mind to ignore. Lifting shape-constraint to +criterion #1 pulls the design-shape question to the front of the +assessment, where it can land on DWIM as the right answer instead of +being skipped past. + +Five outcomes per candidate: + +- **Implement candidate first (flip ordering)** — when the candidate + carries this feature's destination architecture (per 2b), + implementing it first is the right route. This feature exits the + current implementation pass and returns later, anchored on the + candidate's slot-owner shape. Applies *only* to destination- + architecture siblings; for ordinary fold-in candidates, this + outcome doesn't apply. Per the *Order-inversion not surfaced* + failure mode. +- **Fold in (full)** — implement the candidate (or candidate phase) + in this implementation pass. Used when impact is high and scope + fits. +- **Partial implementation** — implement a focused subset of the + candidate in this pass. Concrete code lands; some of the + candidate stays unimplemented. Used when impact is high and a + focused cut exists. +- **Design with in mind** — no new code for the candidate, but this + feature's implementation shape is constrained by the candidate's + eventual needs. Used when impact is medium and the candidate's + shape is enough to inform design but not enough to warrant code. +- **Ignore for now** — defer entirely. Used when impact is low or + the candidate's eventual landing wouldn't meaningfully constrain + this feature's shape. + +Per the *Conflating partial-implementation with design-with-in-mind* +failure mode, the partial vs design-with-in-mind distinction is +load-bearing — partial means code lands; design-with-in-mind means +no code, only shape constraints. + +**Default-recommended outcome per candidate type.** The four-criteria +walk can land anywhere on the outcome spectrum, but candidate type +carries strong priors. Diverging from the default without naming +*which criterion* pushed the assessment is a smell: + +| Candidate type | Default outcome | Notes | +|---|---|---| +| **Destination-architecture sibling** (cluster sibling at `definition: technical`+ carrying this feature's destination slot-owner shape) | Flip ordering **or** Full fold-in | Per *Order-inversion not surfaced* failure mode. Never *ignore* — architecturally wrong by construction. | +| **Primary cluster sibling** | Full fold-in / Partial | Assess via the four criteria. *Ignore* only if low-impact AND low-shape-constraint. | +| **Cross-cluster axis cousin** (e.g., cluster D / E / G / H cousins for a cluster-C feature) | **Design with in mind** | Per *Cross-cluster axis traversal short-circuit* failure mode + `clusters.md` → *Selection / filtering across clusters*. Promote to Partial / Full only if Impact-if-deferred clears the threshold; demote to Ignore only if Shape-constraint-if-deferred is genuinely low. | +| **Use-case composition refactor** | Design with in mind | Let standard composition land cleanly; verify the variant engine composes post-landing. | + +The defaults are starting points. When your assessment lands at a +different outcome, name *which criterion* pushed it (e.g., +"Impact-if-deferred is high, so promoting capability-probing from +DWIM to Partial"). The defaults exist to counter-bias the historical +"implement now / ignore" pull of the four-criteria walk — +particularly for cross-cluster cousins, where the cluster doc +already says DWIM is the typical landing. + +**Value-curve check before recommending scope.** Per `clusters.md`'s +"can-play vs actual support" framing, check whether the +scope-as-recommended produces user-meaningful capability. A cut +that ships only spec-compliance / passive recognition is a signal +that the scope needs revisiting — either fold in more of the +dynamic / interactive phases, or be explicit with the user about +the value-curve constraint. + +**2e. Ask the user to confirm — one question per candidate.** + +**Use `AskUserQuestion`** to present each candidate's recommendation +and let the user confirm or override. One question per candidate; +batch into rounds of up to 4 per `AskUserQuestion` call. For each +candidate's question: + +- **Question text** — name the candidate and summarize the + assessment in one sentence. +- **Options (4)** — four of the outcomes, with the recommended + outcome listed first and labeled "(Recommended)". Order the + remaining three by the next-best fit per the assessment. +- **Description per option** — one-line summary of what that + outcome means for this candidate (concrete: "implement basic + switchAudioQuality alongside" / "leave flush orchestration + codec-change-extensible" / "no change"). + +When the candidate carries destination architecture (per 2b), the +five-outcome list exceeds `AskUserQuestion`'s 4-option cap. Drop +"Ignore for now" — ignoring a destination-architecture sibling is +architecturally-wrong-by-construction. The remaining four (Flip +ordering / Full fold-in / Partial / Design-with-in-mind) fit the +cap. **Ordering surfacing also belongs in Step 2's narrative +report**, not just inside the per-candidate `AskUserQuestion`: +mention the destination-architecture sibling and the flip-ordering +option in prose before the AskUserQuestion call, so the user sees +the framing alongside the recommendation. + +Allow the user to select Other for adjustments not captured by the +listed-outcome shape (e.g., "partially implement but limit to X"). + +After confirmation, the **agreed scope** is the union of: + +- This feature's chosen phases (refined in Step 3). +- Each fold-in / partial-implementation candidate's chosen scope. +- Each "design with in mind" candidate's shape constraint (informs + design without adding code). + +This agreed scope feeds Step 3's open-questions discussion and +final scope confirmation. + +**Stop and report back to the user** with: + +1. The cluster + axis identification (from 2a). +2. The destination architecture sketch (from 2b). +3. The candidate enumeration + per-candidate assessment + + recommended outcome (from 2c-2d). +4. The value-curve check result. +5. The `AskUserQuestion`-confirmed outcomes per candidate (from 2e). +6. The resulting agreed scope. + +### Step 3 — Resolve open questions + confirm final scope + +An **explicit conversational stage** — not optional, not implicit. +After Step 2's fold-in assessment delivers the agreed scope: + +- **Walk through the feature doc's Open questions section** plus any + open questions surfaced by fold-in candidates. Which need to + resolve *before* coding? Which can stay open as known-unknowns? +- **Confirm phase scope at chunk granularity.** Step 2's agreed scope + is the high-level frame; this step decomposes to specific phase + rows + fold-in subsets the implementation will produce. +- **Confirm composition mechanism per chunk** — (i) subtractive (no + new code), (ii) config-driven (existing behavior gains a knob), + (iii) new behavior (route to `/spf-create-behavior`), (iv) + behavior update with purpose change (route to + `/spf-update-behavior`), (v) behavior refactor with preserved + purpose (route to `/refactor-behavior`), (vi) media-layer / + network-layer change (handle inline or defer per the downstream-skill-missing branch). -- **Resolve open questions the implementation needs.** Per the - pre-deciding-things failure mode, only resolve what the implementation - forces; leave the rest as open questions in the doc. +- **Resolve open questions the implementation needs.** Only resolve + what the implementation forces; leave the rest as open questions + in the doc. -**Use `AskUserQuestion`** for clear-cut choices (phase scope, -composition mechanism per chunk, open-question resolutions). +**Use `AskUserQuestion`** for clear-cut choices (open-question +resolutions, composition mechanism per chunk). -### Step 3 — Map phases to implementation chunks +### Step 4 — Map phases to implementation chunks Per the agreed scope, decompose into discrete chunks. Each chunk is: - **Small enough to TDD individually** — one test (or small test set), one implementation file change, one composition wiring tweak. -- **Categorized by composition mechanism** — drives Step 6's routing. +- **Categorized by composition mechanism** — drives Step 7's routing. - **Sequenced for least-risk order** — primitives before behaviors; behaviors before composition wiring; composition wiring before integration tests. @@ -322,7 +736,7 @@ Per the agreed scope, decompose into discrete chunks. Each chunk is: | Create switchAudioQuality behavior | New behavior | `/spf-create-behavior` | `switch-audio-quality.test.ts` (new) | | Wire switchAudioQuality into composition | Composition | None | `engine.test.ts` composition assertion | -### Step 4 — Apply cross-cutting concern checks +### Step 5 — Apply cross-cutting concern checks Run the failure-mode catalog and the feature doc's *Likely cross-cutting impact* section against the chunk list. Each check fires when its signals @@ -344,7 +758,7 @@ are present: catalog, if the chunk touches buffer behavior and codecs change, surface the `changeType()` vs `flushBuffer` question explicitly. -### Step 5 — TDD plan +### Step 6 — TDD plan For each chunk, name: @@ -353,16 +767,16 @@ For each chunk, name: - **The composition wiring change** — if any. - **Acceptance criterion** — what does "done" look like for this chunk? -This output drives Step 6's per-chunk implementation loop. +This output drives Step 7's per-chunk implementation loop. **The TDD plan is the seed of the feature doc's *Verification* -section.** Step 8 persists each chunk's test (file path + test name + +section.** Step 9 persists each chunk's test (file path + test name + assertion summary) into the feature doc — the TDD plan does not live only in chat. Name tests with assertion summaries suitable for the -doc from the start, so Step 8 is a transcription pass rather than a +doc from the start, so Step 9 is a transcription pass rather than a re-articulation. -### Step 6 — Implement (test-first per chunk; route to downstream skills) +### Step 7 — Implement (test-first per chunk; route to downstream skills) Iterate per chunk: @@ -394,7 +808,7 @@ a downstream skill that doesn't exist yet (or is only a stub): The user makes the call. Don't apply ad-hoc discipline silently. -### Step 7 — Final-shape audit (per chunk + cumulative) +### Step 8 — Final-shape audit (per chunk + cumulative) After each chunk: @@ -411,10 +825,10 @@ Cumulative audit after all chunks: *Likely cross-cutting impact* section either addressed or explicitly deferred? - **No silent ad-hoc downstream discipline?** — if any chunk went the - "apply discipline ad-hoc" path in Step 6, the extraction TODO is + "apply discipline ad-hoc" path in Step 7, the extraction TODO is flagged. -### Step 8 — Update feature doc as living artifact +### Step 9 — Update feature doc as living artifact The feature doc transitions from `coarse` → `sketched` (or `technical` → `sketched`) as code lands. Required updates: @@ -432,7 +846,7 @@ The feature doc transitions from `coarse` → `sketched` (or `technical` → lands)** — populated with actual file paths, behavior names, state slots, helpers. See `audio-playback.md` for the canonical shape. - **Verification (required once any phase implementation lands)** — - **this is the persisted TDD artifact.** The Step 5 TDD plan lives + **this is the persisted TDD artifact.** The Step 6 TDD plan lives here in the doc, not just in chat. Each chunk's test gets a line: test file path → test name → assertion summary. Add *Sandbox* entries where demos exist; add *Out of scope / deferred* sub-list @@ -465,9 +879,9 @@ This is **not optional** — the doc-as-living-artifact discipline is load-bearing for the registry staying current. Per the *Status update skipped* failure-mode entry. -### Step 9 — Commit (with user confirmation) +### Step 10 — Commit (with user confirmation) -After Step 7 audit is clean and Step 8 doc update lands: +After Step 8 audit is clean and Step 9 doc update lands: 1. **Audit working-tree state.** `git status -s`. Surface any pre-existing uncommitted work outside the implementation scope. @@ -489,48 +903,73 @@ After Step 7 audit is clean and Step 8 doc update lands: ## Output format -Propose Steps 1–5 outputs as a structured report before writing any code: +Propose Steps 1–6 outputs as structured reports before writing any code, +in order: -1. **Feature identification** (Step 1's report — feature, scope target, - sources, template implementations) -2. **Ambiguities + open questions to resolve** (Step 2) -3. **Chunk decomposition** (Step 3 — chunk list with mechanism + downstream - skill routing) -4. **Cross-cutting concerns** (Step 4) -5. **TDD plan** (Step 5 — per-chunk test + implementation targets) +1. **Feature identification** (Step 1's report — feature, composition + target, sources, template implementations) +2. **Destination architecture + fold-in assessment** (Step 2's report — + cluster/axis identification, destination sketch, candidate + assessment + recommendations, value-curve check, agreed scope after + `AskUserQuestion` confirmations) +3. **Open questions + final scope** (Step 3 — resolved open questions, + final phase-and-fold-in scope at chunk granularity) +4. **Chunk decomposition** (Step 4 — chunk list with mechanism + + downstream skill routing) +5. **Cross-cutting concerns** (Step 5) +6. **TDD plan** (Step 6 — per-chunk test + implementation targets) -After user confirmation, proceed to Step 6 per-chunk loop. Surface Steps -7–9 outputs after implementation. +After user confirmation, proceed to Step 7 per-chunk loop. Surface Steps +8–10 outputs after implementation. ## Why this order Same shape as `/spf-document-feature` and `/spf-document-use-case`. Steps -1–2 force framing before mechanical work. Step 3 commits to a concrete -chunk list; Step 4 runs cross-cutting checks while context is fresh; Step -5 commits to TDD targets. Step 6 produces the artifact; Step 7 audits. -Step 8 keeps the registry current. Step 9 hands the commit boundary to +1–3 force framing before mechanical work. Step 4 commits to a concrete +chunk list; Step 5 runs cross-cutting checks while context is fresh; Step +6 commits to TDD targets. Step 7 produces the artifact; Step 8 audits. +Step 9 keeps the registry current. Step 10 hands the commit boundary to the user. -The novel discipline compared to the document-* skills is the **chunk -decomposition + per-chunk downstream-skill routing** at Steps 3–6 — -implementation work is inherently more granular than documentation work, -and the chunk-level discipline is what keeps it from sprawling. +The novel disciplines compared to the document-* skills: -## Why a discussion stage (not implicit) +- **Step 2's destination-architecture + fold-in assessment** — + implementation choices anchor the architecture more concretely + than documentation choices do; framing the destination + fold-ins + *before* scope-committing prevents local-optimal cuts that don't + fit the cluster's larger shape. +- **Chunk decomposition + per-chunk downstream-skill routing** at + Steps 4–7 — implementation work is inherently more granular than + documentation work, and the chunk-level discipline is what keeps + it from sprawling. -The open-questions-skipped failure mode is the canonical example: feature -docs at coarse depth have unresolved design questions that block -implementation. Resolving them silently via the edit loses the user's -design intent. Step 2's explicit conversational stage forces resolution -in the open, with the user making the call. +## Why two discussion stages (not one) + +This skill has **two** explicit conversational stages: Step 2 (fold-in +assessment, with `AskUserQuestion` per candidate) and Step 3 (open +questions + final scope). They're separated because they resolve +different decisions: + +- **Step 2** decides *which features / phases* the implementation pass + touches. Without this stage, scope-narrow choices that look reasonable + in isolation ship implementations that get reworked when adjacent + features land, or ship capabilities of low user-meaningful value. +- **Step 3** decides *the specifics within the agreed scope* — open + questions, composition mechanisms per chunk, phase-row-level cuts. + Without this stage, the open-questions-skipped failure mode produces + silent design decisions that lose the user's design intent. + +Both stages use `AskUserQuestion`. Resolving them in chat — without the +structured presentation — loses both the agreed scope and the agreed +design decisions. ## When this is the wrong skill - **You want to document a feature (not yet implemented)** → `/spf-document-feature`. - **You want to implement a use-case composition (not a single feature)** → - *(future)* `/spf-implement-use-case`. For now, this skill can be invoked - per-constituent-feature of a use case. + `/spf-implement-use-case`. That skill consumes the use-case doc and + routes per-constituent-feature back into this skill. - **You want to refactor an existing behavior without feature scope** → `/refactor-behavior`. - **You want to split or merge behaviors** → `/refactor-behavior`'s @@ -541,7 +980,7 @@ in the open, with the user making the call. ## How the failure-mode catalog grows Same pattern as other SPF skills: when a new failure mode surfaces during -use (most likely during Step 6 per-chunk implementation or Step 7 audit): +use (most likely during Step 7 per-chunk implementation or Step 8 audit): 1. Add an entry to the *Failure-mode catalog* section above with the risk pattern and a worked-example citation. diff --git a/apps/sandbox/app/shared/sources.ts b/apps/sandbox/app/shared/sources.ts index 12dc1294..ffa30640 100644 --- a/apps/sandbox/app/shared/sources.ts +++ b/apps/sandbox/app/shared/sources.ts @@ -31,6 +31,12 @@ export const SOURCES = { type: 'hls', subType: 'mp4', }, + 'hls-multi-audio': { + label: 'HLS - Multi-language audio', + url: 'https://stream.mux.com/s41JYeqIpBMBzE4OzxDyGR2yrp2hD1CQ6gJN9SlVGDQ.m3u8', + type: 'hls', + subType: 'mp4', + }, 'hls-live': { label: 'HLS - Live Stream Big Buck Bunny', url: 'https://stream.mux.com/v69RSHhFelSm4701snP22dYz2jICy4E4FUyk02rW4gxRM.m3u8', diff --git a/apps/sandbox/templates/spf-segment-loading/index.html b/apps/sandbox/templates/spf-segment-loading/index.html index 9d25805f..7c2c2cd7 100644 --- a/apps/sandbox/templates/spf-segment-loading/index.html +++ b/apps/sandbox/templates/spf-segment-loading/index.html @@ -107,8 +107,9 @@ #throughput-display.has-data { color: #16a34a; } #throughput-display.warming { color: #d97706; } - /* ── Rendition / resolution panels ── */ + /* ── Rendition / resolution / audio panels ── */ #rendition-picker, + #audio-track-picker, #resolution-status { padding: 12px 15px; margin: 16px 0; @@ -117,7 +118,30 @@ border: 1px solid #ddd; border-radius: 4px; } - #rendition-picker h2, #resolution-status h2 { margin-top: 0; font-size: 14px; } + #rendition-picker h2, #audio-track-picker h2, #resolution-status h2 { margin-top: 0; font-size: 14px; } + + #audio-track-buttons { display: flex; flex-wrap: wrap; gap: 8px; } + .audio-track-btn { + padding: 6px 12px; + font-family: monospace; + font-size: 11px; + cursor: pointer; + background: #fff; + border: 2px solid #999; + border-radius: 4px; + } + .audio-track-btn:hover { background: #eee; border-color: #555; } + .audio-track-btn.selected-default { font-weight: bold; background: #e6f0ff; border-color: #0070f3; } + .audio-track-btn.selected-pinned { font-weight: bold; background: #fef3c7; border-color: #d97706; } + + .audio-status { display: flex; gap: 10px; align-items: center; margin-bottom: 10px; } + .mode-default { font-weight: bold; color: #0070f3; } + .mode-pinned { font-weight: bold; color: #d97706; } + .clear-filter-btn { + padding: 4px 10px; font-size: 11px; color: #d97706; cursor: pointer; background: #fef3c7; + border: 1px solid #d97706; border-radius: 4px; + } + .clear-filter-btn:hover { background: #fde68a; } #rendition-buttons { display: flex; flex-wrap: wrap; gap: 8px; } .rendition-btn { @@ -222,6 +246,11 @@
Waiting for presentation…
+
+

Audio Tracks

+
Waiting for presentation…
+
+

Track Resolution Status

Waiting for presentation…
diff --git a/apps/sandbox/templates/spf-segment-loading/main.ts b/apps/sandbox/templates/spf-segment-loading/main.ts index ef1da3f1..10567a10 100644 --- a/apps/sandbox/templates/spf-segment-loading/main.ts +++ b/apps/sandbox/templates/spf-segment-loading/main.ts @@ -17,6 +17,7 @@ const video = document.getElementById('video') as HTMLVideoElement; const logsDiv = document.getElementById('logs') as HTMLDivElement; const stateDiv = document.getElementById('state') as HTMLDivElement; const renditionButtonsDiv = document.getElementById('rendition-buttons') as HTMLDivElement; +const audioTrackButtonsDiv = document.getElementById('audio-track-buttons') as HTMLDivElement; const resolutionListDiv = document.getElementById('resolution-list') as HTMLDivElement; const nowPlayingQualityDiv = document.getElementById('now-playing-quality') as HTMLDivElement; const throughputDiv = document.getElementById('throughput-display') as HTMLDivElement; @@ -62,6 +63,10 @@ function getVideoTracks(presentation: SimpleHlsEngineState['presentation']) { return presentation?.selectionSets?.find((s) => s.type === 'video')?.switchingSets[0]?.tracks ?? []; } +function getAudioTracks(presentation: SimpleHlsEngineState['presentation']) { + return presentation?.selectionSets?.find((s) => s.type === 'audio')?.switchingSets[0]?.tracks ?? []; +} + // ── Display functions ───────────────────────────────────────────────────────── function updateShareUrl() { const p = new URLSearchParams(); @@ -169,6 +174,141 @@ function renderRenditionPicker() { } } +// Signature of the currently-rendered audio track set. Lets the picker rebuild +// its buttons only when the track set actually changes — selection and pin +// changes update existing buttons in place (see renderAudioTrackPicker) so a +// click or hover isn't interrupted by a full DOM teardown. +let audioTrackSetKey = ''; + +function renderAudioTrackPicker() { + if (!engine || !signals) return; + const presentation = engine.state.presentation.get(); + const selectedAudioTrackId = engine.state.selectedAudioTrackId.get(); + const userFilter = engine.state.userAudioTrackSelection.get(); + const tracks = getAudioTracks(presentation); + + if (tracks.length === 0) { + audioTrackButtonsDiv.textContent = presentation ? 'No audio tracks found' : 'Waiting for presentation…'; + audioTrackSetKey = ''; + return; + } + + // The picker pins by language (see the click handler / track-switching + // behavior), so multiple same-language renditions are NOT independently + // selectable — clicking the "med" variant still just pins the language. + // Collapse to one button per selection identity so the UI matches what + // selection actually guarantees. + const groups = getAudioSelectionGroups(tracks); + const setKey = groups.map((group) => group.key).join('|'); + if (setKey !== audioTrackSetKey) { + audioTrackSetKey = setKey; + buildAudioTrackButtons(groups); + } + updateAudioTrackSelection(tracks, selectedAudioTrackId, userFilter); +} + +// One selectable audio identity: a language (when present) or a single track id. +// `key` is exactly what the click handler pins on, so highlighting and dedupe +// share one notion of identity. +interface AudioSelectionGroup { + key: string; + byLanguage: boolean; + language?: string | undefined; + label: string; + members: string[]; +} + +/** Collapse audio tracks to one entry per selection identity (language, else id). */ +function getAudioSelectionGroups(tracks: ReturnType): AudioSelectionGroup[] { + const groups = new Map(); + for (const track of tracks) { + const language = track.language || undefined; + const key = language ?? track.id; + let group = groups.get(key); + if (!group) { + const name = 'name' in track && track.name ? track.name : track.id; + group = { key, byLanguage: !!language, language, label: `${language ?? '—'} · ${name}`, members: [] }; + groups.set(key, group); + } + // Member labels (bitrate, else groupId tier) are surfaced in the tooltip so + // the collapsed renditions are still inspectable. + const groupId = 'groupId' in track ? track.groupId : undefined; + const tier = track.bandwidth ? formatBandwidth(track.bandwidth) : groupId; + group.members.push(tier ?? track.id); + } + return [...groups.values()]; +} + +/** (Re)build the static button list — one per selection group, tagged with its key. */ +function buildAudioTrackButtons(groups: AudioSelectionGroup[]) { + audioTrackButtonsDiv.innerHTML = ''; + + const statusRow = document.createElement('div'); + statusRow.id = 'audio-status-row'; + statusRow.className = 'audio-status'; + audioTrackButtonsDiv.appendChild(statusRow); + + for (const group of groups) { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.dataset.selectionKey = group.key; + // Base label minus the selection badge; the badge is toggled in place. + btn.dataset.label = group.label; + btn.title = group.byLanguage + ? `language: ${group.language} · ${group.members.length} rendition(s): ${group.members.join(', ')}` + : `id: ${group.key}`; + btn.addEventListener('click', () => { + const filter = group.byLanguage ? { language: group.language } : { id: group.key }; + log( + `Audio track filter: ${JSON.stringify(filter)} — mid-stream flush will fire if language differs from buffered`, + 'warning' + ); + signals.state.userAudioTrackSelection.set(filter); + }); + audioTrackButtonsDiv.appendChild(btn); + } +} + +/** Update the status row and per-button selected state without tearing down. */ +function updateAudioTrackSelection( + tracks: ReturnType, + selectedAudioTrackId: string | undefined, + userFilter: SimpleHlsEngineState['userAudioTrackSelection'] +) { + const isPinned = userFilter !== undefined; + + const statusRow = document.getElementById('audio-status-row'); + if (statusRow) { + statusRow.innerHTML = ''; + const modeLabel = document.createElement('span'); + modeLabel.className = isPinned ? 'mode-pinned' : 'mode-default'; + modeLabel.textContent = isPinned ? `🔒 Pinned: ${JSON.stringify(userFilter)}` : '🌐 Default pick'; + statusRow.appendChild(modeLabel); + if (isPinned) { + const clearBtn = document.createElement('button'); + clearBtn.type = 'button'; + clearBtn.className = 'clear-filter-btn'; + clearBtn.textContent = 'Clear filter'; + clearBtn.addEventListener('click', () => { + log('Cleared userAudioTrackSelection (back to default picker)', 'success'); + signals!.state.userAudioTrackSelection.set(undefined); + }); + statusRow.appendChild(clearBtn); + } + } + + // The selected track belongs to a group keyed by its language (else its id); + // highlight that group's button. + const selectedTrack = tracks.find((track) => track.id === selectedAudioTrackId); + const selectedKey = selectedTrack ? selectedTrack.language || selectedTrack.id : undefined; + for (const btn of audioTrackButtonsDiv.querySelectorAll('button[data-selection-key]')) { + const isSelected = btn.dataset.selectionKey === selectedKey; + btn.className = `audio-track-btn${isSelected ? (isPinned ? ' selected-pinned' : ' selected-default') : ''}`; + const badge = isSelected ? (isPinned ? ' 🔒' : ' 🌐') : ''; + btn.textContent = `${btn.dataset.label ?? ''}${badge}`; + } +} + function renderResolutionStatus() { if (!engine) return; const presentation = engine.state.presentation.get(); @@ -333,13 +473,16 @@ function startEngine(src: string) { } }); - // Throughput + rendition picker + resolution status — re-render on any state change - const stopStateUI = effect(() => { - snapshot(engine.state); // track all state changes - updateThroughputDisplay(); - renderRenditionPicker(); - renderResolutionStatus(); - }); + // One effect per UI region, each auto-tracking only the signals its renderer + // reads. The previous single effect snapshotted *all* of engine.state, so + // high-frequency fields (currentTime, bandwidthState) re-fired every renderer + // many times a second — and the picker's full innerHTML rebuild interrupted + // clicks/hover. Now the audio picker re-runs only on presentation / + // selectedAudioTrackId / userAudioTrackSelection changes. + const stopThroughputUI = effect(() => updateThroughputDisplay()); + const stopRenditionUI = effect(() => renderRenditionPicker()); + const stopAudioPickerUI = effect(() => renderAudioTrackPicker()); + const stopResolutionUI = effect(() => renderResolutionStatus()); // Context logger const stopContextLogger = effect(() => { @@ -399,7 +542,10 @@ function startEngine(src: string) { cleanupEffects = () => { stopStateLogger(); - stopStateUI(); + stopThroughputUI(); + stopRenditionUI(); + stopAudioPickerUI(); + stopResolutionUI(); stopContextLogger(); }; diff --git a/internal/design/spf/features/audio-abr.md b/internal/design/spf/features/audio-abr.md index 669f33e9..73f6eb16 100644 --- a/internal/design/spf/features/audio-abr.md +++ b/internal/design/spf/features/audio-abr.md @@ -122,22 +122,18 @@ Things this feature probably forces decisions on, not just additions: sample time), or (b) does the slot hold separate per-source accumulators that compose at read time? The current `BandwidthState` shape would need a small extension either way. -- **`selectedAudioTrackId` triple-writer.** When all three writers - land (default + ABR + multi-language-audio Tier 2), the - characterization is: (a) `selectAudioTrack` writes once on - presentation-resolved entry, default-picker output (config / DOM - decision domain, one-shot trigger, cheap write); (b) - `switchAudioQuality` writes on bandwidth changes (derived decision - domain, ongoing reactive trigger, cheap write); (c) multi-language- - audio Tier 2 writes on consumer programmatic call (intent decision - domain, one-shot trigger, may incur cost via flush). Same coordination - shape as video's `selectedVideoTrackId` (`selectVideoTrack` default - + `switchVideoQuality` + external manual override) — with the added - complication that multi-language-audio's intent write may have - *side effects* (audio-side buffer flush) that pin-to-quality writes - don't. The constraint+filter slot precedent (one writer on the - selection slot, separate filter slots) may apply here too — see - next bullet. +- **`selectedAudioTrackId` writer characterization.** As of multi- + language-audio Tier 2 landing, the slot stays **single-writer** — + `switchAudioTrack` (in `playback/behaviors/dom/switch-audio-track.ts`) + is the sole writer. Consumer intent flows through the + `userAudioTrackSelection` constraint+filter slot. Mid-stream flush is + dispatched from `switchAudioTrack` itself (slot owner = flush dispatcher + — keeps slot writes and consequences co-located, sitting architecturally + between `SegmentLoaderActor` and `SourceBufferActor`). When audio-abr + ships, `switchAudioQuality` extends `switchAudioTrack` with bandwidth + + ABR algorithm — either by rename + extension or as a sibling variant + that supplants it in the engine composition. Either way, the slot stays + single-writer and the filter shape carries over. - **Constraint+filter vs multi-writer for the manual-override slot.** video-abr's `userVideoTrackSelection` slot is a *constraint+filter* (the selection writer reads it as a filter; doesn't write the @@ -192,12 +188,16 @@ Things this feature probably forces decisions on, not just additions: redundant data structure with similar semantics. Decision likely depends on whether the EWMA pattern wants extension to handle mixed-source sampling, or whether two slots is genuinely cleaner. -- **Manual-override slot design.** Single `userAudioTrackSelection` - slot serving both bitrate-pinning (analog of video) AND language- - pinning (multi-language-audio Tier 2), vs separate slots. video-abr - uses one slot for the dual role; audio may differ because - language-switching has side effects (flush) that pin-to-quality - doesn't. Affects the multi-writer count on `selectedAudioTrackId`. +- **Manual-override slot sharing.** `userAudioTrackSelection` slot + exists now (added by multi-language-audio Tier 2) and serves + language-pinning. Open: does audio-abr reuse the same slot for + bitrate-pinning too (one slot, dual role, mirrors video) — likely, + since the partial-track shape supports both. Side-effect concerns + (mid-stream flush) live in `switchAudioTrack`'s flush effect and key + off `initTrackId` vs `selectedAudioTrackId` mismatch — they fire on + language-switch and would naturally also fire on cross-codec + bitrate-switch (whose handling routes to 5.1-surround-selection's + `changeType()` path via a different `flushStrategy`). - **EWMA accumulator design under shared bandwidth state.** Single unified accumulator absorbing both video and audio samples, vs per-source accumulators read with a combining function (e.g., @@ -236,10 +236,16 @@ Things this feature probably forces decisions on, not just additions: - **[audio-playback](./audio-playback.md)** — the single-rendition baseline this feature extends. The "What's not implemented" Audio ABR bullet there is resolved by this doc. -- **[multi-language-audio](./multi-language-audio.md)** *(coarse)* — - prerequisite per Notion #8 for the rendition-group machinery. - Audio-ABR operates within a language; multi-language-audio handles - across-language switching. +- **[multi-language-audio](./multi-language-audio.md)** *(partial, sketched)* — + prerequisite per Notion #8 for the rendition-group machinery, now mostly + implemented. Audio-ABR operates within a language; multi-language-audio + handles across-language switching. Shared infrastructure landed: + `userAudioTrackSelection` filter slot, `switchAudioTrack` behavior + (slot owner; precursor to `switchAudioQuality`), and pluggable + `flushStrategy` config with `nextSegmentBoundaryFlushStrategy` as the + default. Audio-abr Phase 3's `switchAudioQuality` either renames + `switchAudioTrack` + adds bandwidth logic, or coexists with it as a + sibling variant in the engine composition. - **[5.1-surround-selection](./5.1-surround-selection.md)** — consumes its `changeType()` primitive for cross-channel-count switching when audio-ABR wants to cross those boundaries. diff --git a/internal/design/spf/features/audio-playback.md b/internal/design/spf/features/audio-playback.md index 9bee580d..cf2af4ce 100644 --- a/internal/design/spf/features/audio-playback.md +++ b/internal/design/spf/features/audio-playback.md @@ -42,19 +42,16 @@ Capability slices around today's audio playback contract. ## What's not implemented -- **Language-aware default selection** — `pickAudioTrack` (in - `packages/spf/src/media/primitives/select-tracks.ts`) implements the - three-tier picker (`preferredAudioLanguage` → `DEFAULT=YES` → - first-track) and is structurally ready to use, but isn't wired as - the default in `selectAudioTrack`. The `preferredAudioLanguage` - field on `SimpleHlsEngineConfig` is therefore inert with the default - picker — consumers must override via `SelectAudioTrackConfig.picker` - to make it take effect. Wiring `pickAudioTrack` as the default is a - one-line change. -- **Mid-stream audio rendition switching** — covered by - [`multi-language-audio`](./multi-language-audio.md) (coarse). No - programmatic write to `selectedAudioTrackId`, no audio buffer flush - on switch. +- **Multi-rendition recognition + programmatic selection + mid-stream + switching** — covered by + [`multi-language-audio`](./multi-language-audio.md) (partial, sketched — + Tier 1 + most of Tier 2 implemented). Default selection now uses + `pickAudioTrack`'s three-tier picker (`preferredAudioLanguage` → + `DEFAULT=YES` → first-track), so `preferredAudioLanguage` config + takes effect. Tier 2 programmatic selection via + `userAudioTrackSelection` filter and same-codec mid-stream switching + with next-segment-boundary flush also implemented. Persistence and + A/V sync policy refinements deferred. - **Audio ABR** — covered by [audio-abr](./audio-abr.md). `setupAudioBufferActors` uses plain `fetchStream`; the bandwidth-sampling `createTrackedFetch` isn't wired into the audio @@ -106,7 +103,8 @@ loadAudioSegments, | Behavior | File | Responsibility | |---|---|---| -| `selectAudioTrack` | `packages/spf/src/playback/behaviors/select-tracks.ts` | Default audio rendition selection on source load | +| `selectAudioTrack` | `packages/spf/src/playback/behaviors/select-tracks.ts` | Default audio rendition selection on source load. Lifecycle-only; mutually exclusive with `switchAudioTrack` | +| `switchAudioTrack` | `packages/spf/src/playback/behaviors/dom/switch-audio-track.ts` | Filter-reactive slot owner + mid-stream flush dispatcher. **Owned architecturally by [`multi-language-audio`](./multi-language-audio.md)** — composed in both `createSimpleHlsEngine` and `createHlsAudioOnlyEngine` today | | `resolveAudioTrack` | `packages/spf/src/playback/behaviors/resolve-track.ts` | Fetches the selected audio media playlist | | `setupAudioBufferActors` | `packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts` | Audio SourceBuffer + actor setup. **Owned architecturally by `mse-mms-pipeline`** | | `loadAudioSegments` | `packages/spf/src/playback/behaviors/dom/load-segments.ts` | Audio segment loading dispatcher. **Owned architecturally by `buffer-management`** | @@ -115,14 +113,20 @@ loadAudioSegments, | Helper | File | Status | |---|---|---| -| `pickFirstTrackId(presentation, 'audio')` | `packages/spf/src/media/primitives/select-tracks.ts` | **Default picker today** | -| `pickAudioTrack(presentation, config)` | `packages/spf/src/media/primitives/select-tracks.ts` | Language-aware (`preferredAudioLanguage` → `DEFAULT=YES` → first-track). Available but not wired | +| `pickAudioTrack(presentation, config)` | `packages/spf/src/media/primitives/select-tracks.ts` | **Default picker today** — language-aware three-tier (`preferredAudioLanguage` → `DEFAULT=YES` → first-track). Wired by [`multi-language-audio`](./multi-language-audio.md) Tier 1 | +| `pickFirstTrackId(presentation, 'audio')` | `packages/spf/src/media/primitives/select-tracks.ts` | Simple first-track fallback (still available for callers overriding via `SelectAudioTrackConfig.picker`) | **State slots:** -- `selectedAudioTrackId` — single-writer today (`selectAudioTrack`). - Becomes multi-writer when `multi-language-audio` adds the - programmatic selection path. +- `selectedAudioTrackId` — single-writer. Owner depends on which audio- + selection behavior is composed: `selectAudioTrack` (lifecycle-only) or + `switchAudioTrack` (filter-reactive + mid-stream flush; the variant + composed in `createSimpleHlsEngine` and `createHlsAudioOnlyEngine` + today). Becomes `switchAudioQuality`'s responsibility when + [audio-abr](./audio-abr.md) lands (extends `switchAudioTrack`). +- `userAudioTrackSelection` — added by [`multi-language-audio`](./multi-language-audio.md). + Consumer-driven `Partial` filter narrowing the audio + candidate set before `switchAudioTrack`'s picker runs. - Reads `presentation` (audio renditions surface in `presentation.selectionSets`) @@ -175,11 +179,6 @@ Consumers wanting language-aware selection today must override ## Open questions -- **Wire `pickAudioTrack` as the default?** `preferredAudioLanguage` - being exposed but inert is confusing. Was deferral to - `multi-language-audio` deliberate, or is this a missed wiring? A - one-line change in `selectAudioTrack`'s wiring would make the - config field do what its name suggests. - **Audio-only composition guarantees.** The engine *tolerates* audio-only sources today, but is it *designed* for them? The [audio-only-mode-override](../use-cases/audio-only-mode-override.md) @@ -192,9 +191,12 @@ Consumers wanting language-aware selection today must override ## Related features -- **multi-language-audio** *(coarse)* — the future extension covering - multi-rendition surfacing, programmatic selection, and mid-stream - switching. Today's `audio-playback` is the baseline it builds on. +- **[multi-language-audio](./multi-language-audio.md)** *(partial, sketched)* — + the extension covering multi-rendition surfacing (free via parser), + language-aware default selection (wires `pickAudioTrack`), programmatic + selection via `userAudioTrackSelection` filter, and same-codec mid-stream + switching via next-segment-boundary flush. Today's `audio-playback` + is the baseline it builds on. - **subtitles** — parallel structure (default picker + per-type segment loading; same `setupTrackResolution` helper). Text-track selection is user opt-in by default; audio's default selection diff --git a/internal/design/spf/features/clusters.md b/internal/design/spf/features/clusters.md index 262c6fd0..038ea69e 100644 --- a/internal/design/spf/features/clusters.md +++ b/internal/design/spf/features/clusters.md @@ -326,6 +326,26 @@ A side-effecting fetch (or segment-load) wrapper produces signals another featur --- +### Selection / filtering across clusters + +The end-to-end "filter, prioritize, select among track candidates" axis. The selection lifecycle for any track type (audio / video / text) crosses cluster C plus four neighbor clusters whose features participate at distinct points in the pipeline: + +| Cluster | Role in selection | Example features | +|---|---|---| +| **C — Track & variant registry** | Owns the slot (`selected*TrackId`) and the picker that chooses among candidates. | `selectAudioTrack`, `switchVideoQuality`, `audio-playback`, `multi-language-audio`, `video-abr`, `audio-abr` | +| **D — Capability probing** | Filters the candidate set *before* selection runs. CODECS / `isTypeSupported` / `changeType()` gating shrinks the set the picker operates over. | `capability-probing`, `5.1-surround-selection` (consumer), `hevc-variant-selection` (consumer) | +| **E — Selection policy** | Caps and modes that *bias* the candidate set. Layered on top of the capability-filtered set; same filter shape, different motivation (config vs platform). | `rendition-selection-caps` | +| **G — Selection resilience** | Alternate URI rotation *within* the selected track. Swaps URIs for the same selection without changing which track is selected; wraps fetch primitives consumed by track resolution and segment loading. | `multi-cdn-failover`, `content-steering`, `network-resilience` | +| **H — Encrypted media (DRM)** | Key-system support *gates which tracks can be selected*. Similar to capability-probing but along a different axis (key system rather than codec). | `drm-support` | + +**Skill action when this pattern is suspected.** A feature in cluster C nearly always interacts with the four neighbor clusters above. When enumerating fold-in candidates in `/spf-implement-feature` Step 2c, walk all four explicitly — even when the feature doc's *Related features* list names only a subset. Candidates often land as "design-with-in-mind" rather than full fold-in (each cluster owns its own primitives), but surfacing them keeps the cluster-C feature's shape from painting into a corner. + +Worked example: `multi-language-audio` should surface `capability-probing` (codec-filter the audio rendition list), `rendition-selection-caps` (audio caps as a future bias), `multi-cdn-failover` (alternate URI rotation per language playlist), and `drm-support` (per-language key-system filtering) as fold-in candidates — even when the feature doc's *Related features* lists only `capability-probing`. The likely outcomes are *design-with-in-mind* / *ignore for now* for most; the discipline is to *surface and assess* rather than silently omit. + +**The destination-architecture sibling shape.** Within cluster C itself, one sibling often carries the destination slot-owner shape that other siblings anchor on (e.g., `switchVideoQuality` for video selection; `switchAudioQuality` per `audio-abr` Phase 3 for audio selection). When implementing a feature that writes the same slot, anchor on the destination sibling's shape — not on whatever extends-the-current-code-shape happens to be available. The "implement the destination sibling first" ordering is a real fold-in outcome; see `/spf-implement-feature` Step 2's *Order-inversion not surfaced* discipline. + +--- + ## Where this doc fits Read alongside: diff --git a/internal/design/spf/features/multi-language-audio.md b/internal/design/spf/features/multi-language-audio.md index 71026be0..e79aedd7 100644 --- a/internal/design/spf/features/multi-language-audio.md +++ b/internal/design/spf/features/multi-language-audio.md @@ -1,79 +1,173 @@ --- -status: draft -date: 2026-05-20 -definition: coarse +status: partial +date: 2026-05-25 +definition: sketched --- # Multi-language audio -Recognize multiple audio renditions from a multivariant HLS playlist, expose them with language metadata, apply a default-selection picker, and support user / programmatic switching of the active audio track. Today the engine plays at most one audio track per source — the one chosen by `selectAudioTrack`'s default picker. Adding this feature is the canonical next step for the track-selection / filtering work. +Recognize multiple audio renditions from a multivariant HLS playlist, expose them with language metadata, apply a default-selection picker, and support user / programmatic switching of the active audio track — including mid-stream switching with audio-buffer flush at the next-segment boundary. ## Status -- **Composition:** not implemented in `createSimpleHlsEngine`. Today's [`audio-playback`](./audio-playback.md) feature is the single-rendition baseline; this feature extends it with multi-rendition recognition, programmatic selection, and mid-stream switching. The foundation behaviors (`selectAudioTrack`, `resolveAudioTrack`, `setupAudioBufferActors`, `loadAudioSegments`) all exist and ship as part of `audio-playback`, but assume a single audio playlist throughout the source's lifetime. -- **Definition depth:** coarse — scope and relations identified; implementation approach not yet sketched. +- **Composition:** Tier 1 (recognition + default selection) and most of Tier 2 (programmatic selection via `userAudioTrackSelection` + mid-stream switching) implemented in `createSimpleHlsEngine` and `createHlsAudioOnlyEngine`. A/V sync policy on mid-stream switch is "switch at next-segment boundary" (starting point, area for improvement). Persistence (Tier 2 phase 6) deferred. +- **Definition depth:** sketched — implementation surface populated for Tier 1 and Tier 2; specific phases (A/V sync policy refinement, persistence) remain coarse pending follow-up. ## Phases of complexity -Following the Tier 1 / Tier 2 framing from the broader inventory: - -| Phase | What | Notes | +| Phase | What | Status | |---|---|---| -| **Tier 1 — Recognition + exposure** | Parser surfaces all audio renditions with `LANGUAGE`, `NAME`, `DEFAULT`, `AUTOSELECT`, `URI` metadata; engine state exposes the candidate list | Pre-req for everything below. Probably free given the subtitles parser pattern — same multivariant-playlist code path | -| **Tier 1 — Default selection** | Three-tier picker: `preferredAudioLanguage` → `DEFAULT=YES + AUTOSELECT=YES` → fallback | Direct parallel to subtitles `selectTextTrack` picker — likely lifts the same shape | -| **Tier 2 — Programmatic selection** | Consumer writes `userAudioTrackSelection` (parallel to `userVideoTrackSelection`) to override default | Requires multi-writer state coordination on `selectedAudioTrackId` (`selectAudioTrack` writes default; programmatic write overrides) | -| **Tier 2 — Mid-stream switching** | When `selectedAudioTrackId` changes mid-playback: flush stale audio range from the existing SourceBuffer (`remove(playhead, end)`), re-resolve the new track's playlist if not already fetched, restart segment loading from current playhead | Same-codec switch (the typical case for language-only renditions) — no SourceBuffer recreation, no `changeType()`. Closest precedent in the codebase: video ABR also keeps the same SourceBuffer and feeds it different-bitrate segments; audio adds the wrinkle that the new segments come from a different media playlist. Codec-change switching (e.g., stereo AAC → 5.1 AC-3 in a different rendition group) is a separate concern handled under `5.1-surround-selection` | -| **Tier 2 — A/V sync during switch** | Hold playback continuity while audio buffer is repopulated; avoid audio gap that exceeds tolerance | Open: do we pause? do we silence-pad? do we accept brief gaps? | -| **Tier 2 — Persistence** | Remember the user's last audio-track choice across sources or sessions | Lower priority; pure policy on top of programmatic API | +| **Tier 1 — Recognition + exposure** | Parser surfaces all audio renditions with `LANGUAGE`, `NAME`, `DEFAULT`, `AUTOSELECT`, `CHANNELS`, `URI`, `CODECS` metadata; engine state exposes the candidate list via `presentation.selectionSets`. | **Implemented** (free — owned architecturally by `parseMultivariantPlaylist` and surfaced via `audio-playback`) | +| **Tier 1 — Default selection** | Three-tier picker: `preferredAudioLanguage` → `DEFAULT=YES` → first track. | **Implemented** — `pickAudioTrack` wired as default in `selectAudioTrack`'s `AUDIO_TRACK_SELECTION_CONFIG` (and shared with `switchAudioTrack`). `preferredAudioLanguage` config field active (was inert before this feature) | +| **Tier 2 — Programmatic selection** | Consumer writes `userAudioTrackSelection` (`Partial`) to narrow the candidate set; `switchAudioTrack` reads the filter, re-fires on changes, short-circuits picker when filter narrows to single track, falls back to unfiltered candidates when filter excludes all. | **Implemented** — sibling of `userVideoTrackSelection`; constraint+filter shape matches video precedent. Slot ownership moved from `selectAudioTrack` to **new behavior `switchAudioTrack`** (precursor to audio-abr's `switchAudioQuality`) | +| **Tier 2 — Mid-stream switching** | When the audio segment loader receives a `load` for a different track than the one currently buffered, `planTasks` compares the new track's language to the buffered track's language (captured on `SourceBufferActor.context.initTrackLanguage`). On mismatch, the loader emits a `{ type: 'remove', start: nextBoundary, end: Infinity }` task *before* the new `append-init` / `append-segment` tasks. Same split as video: `switchVideoQuality` writes the slot; `segment-loader` plans the buffer ops. Loader replan happens naturally via `loadAudioSegments` reactivity to `selectedAudioTrackId`. | **Implemented** — language-predicate today (covers multi-language audio + text language switching; video ABR is unaffected because video tracks have no language). Cross-codec switching (e.g., AAC → AC-3) is a separate predicate handled by 5.1-surround-selection (DWIM — see Stage 2 below) | +| **Tier 2 — A/V sync during switch** | Flush range is "next segment boundary at/after playhead." Current segment plays through; new rendition starts at boundary. | **Implemented (starting point)** — area for improvement: smoother sync via pause / silence-pad alternatives. Stage 2 lifts the predicate / strategy to a per-actor config (see Open questions) | +| **Tier 2 — Persistence** | Remember user's last audio-track choice across sources or sessions. | **Deferred** — policy on top of API, not core to feature mechanism. Future work | ## What's in scope vs out of scope -**In scope:** -- All phases above for HLS VoD content +**In scope (implemented):** +- All Tier 1 and Tier 2 phases above except persistence - HLS spec compliance for `EXT-X-MEDIA:TYPE=AUDIO` rendition handling +- VoD content **Out of scope (separate candidate features):** -- **[audio-abr](./audio-abr.md)** — bandwidth-driven switching within an audio rendition group. Today no audio quality switching exists at all (audio segment loader uses plain `fetchStream`, not `createTrackedFetch`). Audio ABR depends on multi-language audio for the rendition-group machinery but is a distinct feature. -- **5.1-surround-selection** — capability-gated codec selection for audio. Layers on top of multi-language audio's rendition surfacing. Also owns the codec-change switching case (`SourceBuffer.changeType()` or buffer recreation), since cross-codec switches are where the SourceBuffer itself needs to mutate. -- **[audio-only-mode-override](../use-cases/audio-only-mode-override.md)** *(use case; Phase 1 landed)* — engine variant for audio-only delivery. Different composition concern; this feature is about audio-track *selection*, not whether video is present. When this feature lands, the use case composes it for multi-language audio support within the audio-only variant (use case Phase 2). +- **[audio-abr](./audio-abr.md)** *(documented; not implemented)* — bandwidth-driven switching within an audio rendition group. The slot writer `switchAudioQuality` will replace `selectAudioTrack` when audio-abr ships, mirroring video's `selectVideoTrack` / `switchVideoQuality` precedent. `userAudioTrackSelection` filter shape is shared between this feature and audio-abr (single slot serves both bitrate-pinning and language-pinning, per audio-abr's open question pending resolution). +- **[5.1-surround-selection](./5.1-surround-selection.md)** *(coarse, not yet documented)* — capability-gated codec selection for audio. Layers on top of multi-language-audio's rendition surfacing. Owns codec-change switching (cross-codec `changeType()` or buffer recreation). +- **[audio-only-mode-override](../use-cases/audio-only-mode-override.md)** *(use case; Phase 2 partial — this feature composes in as Phase 2 constituent)* — engine variant for audio-only delivery. Different composition concern. **Out of scope (different architectural layer):** -- **DOM `HTMLMediaElement.audioTracks` exposure** — mirroring `selectedAudioTrackId` into `HTMLMediaElement.audioTracks` (parallel to how `syncTextTracks` mirrors text-track selection into the DOM via `` children) is **not** an SPF concern. Browser-native audio-track UI is uneven (especially Safari), and the API surface is consumer-facing. SPF keeps audio-track selection purely state-driven; an adapter or above-the-engine layer may implement something roughly conforming to this API if needed. +- **DOM `HTMLMediaElement.audioTracks` exposure** — mirroring `selectedAudioTrackId` into `HTMLMediaElement.audioTracks` is not an SPF concern. Adapter / above-the-engine layer may implement. +- **Persistence** — deferred. See *Phases of complexity*. ## Likely cross-cutting impact -Things this feature probably forces decisions on, not just additions: +Resolved during implementation: -- **Track registry primitive** — `selectedAudioTrackId` becomes multi-writer (default + programmatic). Today `selectedTextTrackId` is the only multi-writer track-id slot. Two data points may be enough to extract a shared primitive. See `track-registry-primitive` (candidate feature). -- **`resolveAudioTrack` re-resolution** — currently resolves the selected track once per source. Mid-stream switch to a different language means the newly-selected track's media playlist may not yet be fetched — the behavior needs to handle re-resolution when `selectedAudioTrackId` changes mid-presentation. -- **Audio SourceBuffer flush on switch** — the architecturally novel piece. Same-codec language switching does **not** require recreating the SourceBuffer or re-entering `setupAudioBufferActors`'s setup. What's needed is a flush mechanism: `SourceBuffer.remove(playhead, end)` to clear the now-stale audio range, then append from the new rendition. The `SourceBufferActor` already accepts a `remove` message backed by the `flushBuffer` helper (see [`mse-mms-pipeline.md`](./mse-mms-pipeline.md)); what's missing is a behavior that *orchestrates* flush on `selectedAudioTrackId` change. This orchestration is part of this feature's Tier 2 mid-stream-switching phase, not a separately-scoped buffer-flushing feature — primitives in [mse-mms-pipeline.md](./mse-mms-pipeline.md) / [buffer-management.md](./buffer-management.md), orchestration belongs here. -- **`loadAudioSegments` replan on track change** — segment loader currently replans on `currentTime` / `preload` / `loadActivated` changes. Needs to detect `selectedAudioTrackId` change as a replan trigger too. Signal-driven re-eval likely gets most of the way; open question is whether the loader actor's continue/preempt logic handles "different rendition, same buffer" cleanly or whether it treats the new rendition as a fresh source. -- **Manifest parser** — confirm audio renditions surface with the same per-track metadata as subtitles (language, default, autoselect). If they do, Tier 1 is largely a copy of the subtitles selection path; if not, parser work is on the critical path. +- **Track registry primitive** — no extraction. The audio multi-writer pattern (Tier 2 programmatic) uses **constraint+filter** (single slot writer, `userAudioTrackSelection` is consumer intent), not multi-writer. `selectedAudioTrackId` stays single-writer. Premature to extract a shared abstraction at 2 data points (text + audio); text uses multi-writer-with-intent (different shape). +- **`resolveAudioTrack` re-resolution** — no extension needed. `setupTrackResolution`'s `'presentation-resolved'` state effect reads `selectedAudioTrackId` reactively; mid-stream switch fires the effect, scheduling a fetch task for the new track. Source-identity state-exit cleanup cancels stale tasks via `runner.abortAll`. +- **Audio buffer flush placement** — lives at the **`segment-loader` actor's `planTasks`** (between `SegmentLoaderActor` "what to load" and `SourceBufferActor` "remove / append" intelligence), NOT in `switchAudioTrack` (slot owner) or in `setupAudioBufferActors` (actor lifecycle). Same split as the video pipeline: `switchVideoQuality` writes the slot; `segment-loader`'s `planTasks` decides flush + init + segments. For audio, `planTasks` compares `actorCtx.initTrackLanguage` (captured from `append-init` meta) to the new `track.language` — on mismatch, emits a `remove` task spanning next-segment-boundary → Infinity ahead of init / segment tasks. Predicate is **inline today** (language-change). Stage 2 lifts it to a per-actor config (`flushPredicate?` or `flushStrategy?` at `createSegmentLoaderActor` time) for cross-codec, viewport-driven, and other variants. +- **`SourceBufferActor.context` extended with `initTrackLanguage`** — captured from `AppendInitMessage.meta.language` on each `append-init` commit. The downstream consumer is `segment-loader`'s `planTasks`. +- **`loadAudioSegments` replan** — no change needed. Already reactive to `selectedAudioTrackId` via segment loader actor's internal effect. +- **Manifest parser** — already surfaces all needed per-track metadata (verified via existing `audio-playback` Phase 1). ## Open questions -- **A/V sync policy during switch** — pause / silence-pad / accept-gap. May be a config knob. -- **What level of track-registry primitive, if any, does adding the second concordant multi-writer slot force?** Today only text uses orthogonal multi-writer (`selectTextTrack` default + `syncTextTracks` user-action). Audio multi-writer (`selectAudioTrack` default + programmatic write) would be the second concordant data point. Options range from a shared picker helper (e.g., a `pickByLanguageDefaultAutoselect` parameterized by track-type) → a multi-writer coordination utility → a unified track model across audio/video/text. Note: 5.1 / HEVC variant selection won't help decide this — those follow video's constraint+filter pattern, not multi-writer — so this is a text+audio decision, not a wait-for-third-use-case decision. +- **Stage 2: pluggable flush predicate / strategy.** Stage 1 inlines the language-change predicate in `segment-loader`'s `planTasks`. Stage 2 lifts it to a per-actor config at `createSegmentLoaderActor` time. Shape options: + - `flushPredicate?: (prev: BufferedTrackMeta, next: Track) => boolean` — simplest; per-loader policy decision. + - `flushStrategy?: { predicate, range }` — separate decision-of-whether and decision-of-where. + - A/V sync policy alternatives (pause-then-resume, silence-pad) — currently next-segment-boundary is hardcoded. + - Codec-change branching when 5.1-surround-selection lands (different predicate; `changeType()`-aware range). +- **Persistence model.** When implemented: localStorage / cookie / consumer-supplied storage? Across-source preservation rules? Likely adapter-layer, not engine. +- **`userAudioTrackSelection` slot sharing with `switchAudioQuality`.** When audio-abr lands, does the slot serve both language-pinning and bitrate-pinning (video's pattern) or do they split? Resolution deferred to audio-abr implementation. + +## Resolved during this phase's implementation + +- **A/V sync policy** — chose **next-segment-boundary** as starting point (smoother than playhead-forward flush; cheaper than pause / silence-pad). Stage 2 will lift this to a configurable strategy. +- **Programmatic write path** — chose **constraint+filter via `userAudioTrackSelection`** over direct multi-writer write to `selectedAudioTrackId`. Aligns with video's `userVideoTrackSelection` precedent; preserves audio-abr's path to introducing `switchAudioQuality` as the slot owner. +- **Mid-stream flush placement** — flush logic lives in **`segment-loader`'s `planTasks`** (between actor planning and source-buffer execution), NOT in `switchAudioTrack` (slot owner) or `setupAudioBufferActors` (actor lifecycle). This mirrors the video pipeline's split: slot owner writes; loader decides remove / init / segment plan. Two earlier iterations bolted flush onto the wrong host: first `setupAudioBufferActors` (lifecycle violation), then `switchAudioTrack` (blended selection + flush). The final placement keeps `switchAudioTrack` purely a selection-ownership behavior and concentrates buffer-orchestration knowledge in the segment-loader where init / append / remove already get planned together. +- **Flush predicate** — Stage 1 inline check: `actorCtx.initTrackLanguage !== track.language`. Naturally covers audio language switching AND text language switching; video ABR is unaffected (video tracks have no language). Stage 2 lifts to a per-actor predicate / strategy config at `createSegmentLoaderActor` time so cross-codec (5.1 surround), per-channel-count, and other cross-rendition shapes can plug in without modifying `planTasks`. +- **Abstraction shape for `switchAudioTrack`** — chose to **mirror `switchVideoQuality`'s abstraction shape** (separate helper `setupAudioTrackSwitching`, same generic-parameter pattern over selection key + user-selection key + track type, same `getTracks` / `selectOptimal` / `picker` config points) rather than (a) reusing `setupQualitySwitching` directly or (b) shipping an ad-hoc audio-specific structure. The mirrored shape sets up the convergence path: when audio-abr ships, the two helpers either merge (shared generics with optional bandwidth) or stay parallel with minimal divergence. Audio's `selectOptimal` is `selectAudioCurrent` (pin-to-current); audio-abr swaps it for a bandwidth-driven variant matching the video signature. +- **Track-registry primitive extraction** — chose **defer**. Two data points (text + audio) with different multi-writer shapes don't justify extraction yet. + +## Implementation surface + +**Composition:** `packages/spf/src/playback/engines/hls/engine.ts` (default) + `packages/spf/src/playback/engines/hls/engine-audio-only.ts` (variant). Both replace `selectAudioTrack` with `switchAudioTrack` to get filter reactivity + mid-stream flush; engines that want only default-pick-on-load (test setups, future variants) keep composing `selectAudioTrack` (it remains exported, mutually exclusive with `switchAudioTrack`). + +**Behaviors:** + +| Behavior / Actor | File | Responsibility | +|---|---|---| +| `switchAudioTrack` *(new behavior)* | `packages/spf/src/playback/behaviors/switch-audio-track.ts` | **Slot owner for `selectedAudioTrackId`**, filter-reactive (consumes `userAudioTrackSelection`). Single effect in `'presentation-resolved'` that mirrors `switchVideoQuality`'s pattern: filter narrow → single-candidate short-circuit → initial picker → `selectOptimal`. Uses helper `setupAudioTrackSwitching` whose abstraction shape parallels `setupQualitySwitching` (generic over selection key + user-selection key + track type + `getTracks` + `selectOptimal` + `picker`). Pure selection ownership; no flush concern. Path to `switchAudioQuality`: audio-abr Phase 3 swaps `selectAudioCurrent` (pin-to-current) for a bandwidth-driven `selectOptimal` | +| `segment-loader` actor `planTasks` *(extended)* | `packages/spf/src/playback/actors/dom/segment-loader.ts` | Already handled init / append / forward+back-flush task planning. **New Stage-1 predicate**: when `actorCtx.initTrackId !== track.id` AND `actorCtx.initTrackLanguage !== track.language`, emits a `{ type: 'remove', start: nextBoundary, end: Infinity }` task at the front of the task list (before `append-init` + `append-segment`). Generic enough to cover audio-language and text-language switches; video ABR doesn't trigger it (no language attribute). Includes `language` in the emitted `append-init` meta so downstream tracking can compare next switch | +| `SourceBufferActor` *(extended)* | `packages/spf/src/playback/actors/dom/source-buffer.ts` | Context now tracks `initTrackLanguage?` alongside `initTrackId`. Captured from `AppendInitMessage.meta.language` on commit; read by `segment-loader`'s `planTasks` to detect cross-language switches | +| `selectAudioTrack` *(unchanged purpose)* | `packages/spf/src/playback/behaviors/select-tracks.ts` | Lifecycle-only default selection on `presentation-resolved` entry; clears on src unload. Uses `pickAudioTrack` (3-tier) as default picker. Mutually exclusive with `switchAudioTrack` — engines compose one or the other | +| `setupAudioBufferActors` *(unchanged)* | `packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts` | Per-source audio `SourceBufferActor` + `SegmentLoaderActor` lifecycle | + +**State slots:** + +- `selectedAudioTrackId` — single-writer (`switchAudioTrack` when composed; `selectAudioTrack` when the lighter variant is composed instead — they're mutually exclusive). Constraint+filter pattern keeps writer count at 1; intent flows through `userAudioTrackSelection`. +- `userAudioTrackSelection` — new slot in `SimpleHlsEngineState` + `SimpleHlsAudioOnlyEngineState`. `Partial` shape. Single-writer (external consumer via `shareSignals`). Read by `switchAudioTrack`. + +**Actor state:** + +- `SourceBufferActorContext.initTrackLanguage` — new field; captured from `AppendInitMessage.meta.language` on commit. Read by `segment-loader`'s `planTasks`. +- `AppendInitMessage.meta.language` — new field on the message type. Carries language from `planTasks` (which reads `track.language`) to the source-buffer actor. + +**Helpers:** + +| Helper | File | Status | +|---|---|---| +| `setupAudioTrackSwitching` *(new)* | `packages/spf/src/playback/behaviors/switch-audio-track.ts` | Filter-reactive slot-management reactor. Mirrors `setupQualitySwitching`'s abstraction shape (generic over selection key + user-selection key + track type, with `getTracks` / `selectOptimal` / `picker` config). Today's audio-only consumer is `switchAudioTrack`; when audio-abr ships, `switchAudioQuality` either swaps its `selectOptimal` for a bandwidth-aware variant or the two helpers merge into a shared `setupQualitySwitching` | +| `pickAudioTrack` | `packages/spf/src/media/primitives/select-tracks.ts` | **Now wired** as the default picker in both `selectAudioTrack` and `switchAudioTrack` (was inert; required custom-picker override before this feature) | + +**Composition wiring:** Both engine factories swap `selectAudioTrack` for `switchAudioTrack` in their behavior list. Engine state types in both `engine.ts` and `engine-audio-only.ts` gain `userAudioTrackSelection`. + +## Verification + +**Unit tests** (`packages/spf/src/playback/behaviors/tests/select-tracks.test.ts` — `selectAudioTrack` describe block, lifecycle-only variant): + +- `picks track matching preferredAudioLanguage when supplied` — Tier 1 default selection via language preference +- `falls back to DEFAULT=YES track when preferredAudioLanguage does not match` — Tier 1 second-tier fallback +- `falls back to first track when no language preference and no DEFAULT track` — Tier 1 final fallback + +**Unit tests** (`packages/spf/src/playback/behaviors/tests/switch-audio-track.test.ts` — slot-owner variant; Tier 2): + +- *Selection lifecycle:* + - `selects the first audio track when no preference or filter` + - `picks track matching preferredAudioLanguage when supplied` + - `clears selectedAudioTrackId on src unload` +- *Filter reactivity (`userAudioTrackSelection`):* + - `narrows candidates by filter (language)` — Tier 2 programmatic write + - `re-picks on filter change mid-presentation` — Tier 2 reactive re-pick + - `filter narrowing to a single track short-circuits the picker` — single-candidate short-circuit + - `empty filter result falls back to unfiltered candidate set` — graceful no-match fallback + +**Unit tests** (`packages/spf/src/playback/actors/dom/tests/segment-loader.test.ts` — cross-rendition flush predicate): + +- `dispatches \`remove\` from next segment boundary on audio language switch` +- `does not dispatch cross-rendition flush when languages match (audio-abr-style switch)` +- `does not dispatch cross-rendition flush on initial load (no prior initTrackId)` +- `captures language into append-init meta for downstream tracking` + +**Unit tests** (`packages/spf/src/playback/actors/dom/tests/source-buffer.test.ts` — language tracking): + +- `captures initTrackLanguage from append-init meta for downstream cross-rendition flush detection` +- `leaves initTrackLanguage undefined when append-init meta omits language (video)` + +**Unit tests** (`packages/spf/src/playback/engines/hls/tests/engine-audio-only.test.ts`): + +- `exposes userAudioTrackSelection slot for multi-language-audio Tier 2 writes` — variant engine state exposes the filter slot + +**Out of scope / deferred:** + +- End-to-end mid-stream-switch verification (browser-level rendition switching) deferred to manual sandbox / E2E pass once a multi-language test source is available. +- Persistence — not implemented; no test coverage. +- A/V sync policy alternatives (pause / silence-pad) — not implemented. ## Related features -- **audio-playback** — the single-rendition baseline this feature extends. Recognition + default selection at source load already exist there; this feature adds the multi-rendition + switching layer on top. -- **subtitles** — direct template for the selection-picker shape; multi-writer state slot pattern. -- **video-abr** — `userVideoTrackSelection` constraint pattern; precedent for consumer-driven track override coexisting with engine-driven selection. -- **mse-mms-pipeline** — owns the audio `SourceBufferActor` and the `remove` / `flushBuffer` primitives that mid-stream language switching builds on; the lifecycle stays put (same-codec, no recreation), and this feature adds the flush orchestration on top. -- **buffer-management** — audio segment loading uses the same gate shape as video and text; mid-stream switching will push on the planner's track-switch handling (no flush today; same-codec dedup is the current strategy). -- **track-registry-primitive** *(coarse, not yet documented)* — multi-language audio is likely the second forcing data point. First is text-track multi-writer; this is audio multi-writer with mid-stream switching as an added complication. -- **[audio-abr](./audio-abr.md)** — built on top of multi-language audio's rendition surfacing. -- **5.1-surround-selection** *(coarse, not yet documented, candidate)* — capability-gated extension. -- **[audio-only-mode-override](../use-cases/audio-only-mode-override.md)** *(use case; Phase 1 landed)* — engine variant; orthogonal but composition-relevant (Phase 2 of the use case composes this feature for multi-language audio). -- **capability-probing** *(candidate)* — Tier 2 mid-stream codec switching (e.g., stereo AAC → 5.1 AC-3) depends on `changeType()` capability probing surfaced by that feature. +- **[audio-playback](./audio-playback.md)** — single-rendition baseline this feature extends. The "Language-aware default selection" gap there is now resolved. +- **[subtitles](./subtitles.md)** — direct template for the selection-picker shape; multi-writer state slot pattern. Subtitles uses orthogonal multi-writer (`selectTextTrack` + DOM `change`); audio uses constraint+filter — different shapes. +- **[video-abr](./video-abr.md)** — `userVideoTrackSelection` constraint+filter precedent. Same shape as audio's new `userAudioTrackSelection` slot. +- **[audio-abr](./audio-abr.md)** *(documented; pending implementation)* — destination-architecture sibling. When implemented, `switchAudioQuality` will replace `selectAudioTrack` as the `selectedAudioTrackId` writer; `userAudioTrackSelection` filter shape carries over. +- **[5.1-surround-selection](./5.1-surround-selection.md)** *(coarse, not yet documented, candidate)* — codec-change extension. Tier 2 mid-stream flush is designed extensible to codec-change routing. +- **[mse-mms-pipeline](./mse-mms-pipeline.md)** — owns the audio `SourceBufferActor` and the `remove` / `flushBuffer` primitives that mid-stream flush builds on. +- **[buffer-management](./buffer-management.md)** — audio segment loading already replans on `selectedAudioTrackId` change via natural reactivity; no extension needed. +- **[audio-only-mode-override](../use-cases/audio-only-mode-override.md)** *(use case; Phase 2 partial)* — this feature composes in for multi-language audio selection within the audio-only variant. ## Use cases that compose this feature -- **[`audio-only-mode-override`](../use-cases/audio-only-mode-override.md)** *(coarse)* — Phase 2 constituent. When multi-language-audio is implemented, the audio-only delivery variant composes it for language selection within the audio-only variant (e.g., a podcast mode for a multi-language source). Used as-is. +- **[`audio-only-mode-override`](../use-cases/audio-only-mode-override.md)** *(Phase 2 partial — landed with this feature)* — variant engine `createHlsAudioOnlyEngine` exposes `userAudioTrackSelection` and composes the filter-reactive `selectAudioTrack` + flush-aware `setupAudioBufferActors` unchanged from the default engine. ## See also -- `internal/design/spf/features/subtitles.md` — closest analog for the recognition + selection shape -- `internal/design/spf/features/video-abr.md` — `userVideoTrackSelection` constraint pattern +- [subtitles.md](./subtitles.md) — closest selection-shape analog +- [video-abr.md](./video-abr.md) — `userVideoTrackSelection` constraint+filter precedent +- [audio-abr.md](./audio-abr.md) — destination-architecture sibling for `selectedAudioTrackId` - [conventions/signals.md](../conventions/signals.md) — multi-writer slot conventions +- [conventions/behaviors.md](../conventions/behaviors.md) — per-type specialization +- [packages/spf/src/playback/behaviors/select-tracks.ts](../../../../packages/spf/src/playback/behaviors/select-tracks.ts) — `selectAudioTrack` (lifecycle-only variant) +- [packages/spf/src/playback/behaviors/switch-audio-track.ts](../../../../packages/spf/src/playback/behaviors/switch-audio-track.ts) — `switchAudioTrack` (slot owner with filter reactivity) + `setupAudioTrackSwitching` helper (mirrors `setupQualitySwitching`'s shape) +- [packages/spf/src/playback/actors/dom/segment-loader.ts](../../../../packages/spf/src/playback/actors/dom/segment-loader.ts) — `planTasks` cross-rendition flush predicate (Stage 1: language-change inline) +- [packages/spf/src/playback/actors/dom/source-buffer.ts](../../../../packages/spf/src/playback/actors/dom/source-buffer.ts) — `initTrackLanguage` context field + `AppendInitMessage.meta.language` diff --git a/internal/design/spf/use-cases/audio-only-mode-override.md b/internal/design/spf/use-cases/audio-only-mode-override.md index 1a45e42a..15df2579 100644 --- a/internal/design/spf/use-cases/audio-only-mode-override.md +++ b/internal/design/spf/use-cases/audio-only-mode-override.md @@ -149,13 +149,16 @@ Phase 1 baseline: - **[`buffer-management`](../features/buffer-management.md)** — used as-is in Phase 1; Phase 3 surfaces alternative defaults. -Phase 2 (when these features land): +Phase 2: -- **[`audio-abr`](../features/audio-abr.md)** — when implemented, composed in - for multi-bitrate audio. Used as-is. -- **[`multi-language-audio`](../features/multi-language-audio.md)** — when - implemented, composed in for mixed sources with multi-language audio. Used - as-is. +- **[`audio-abr`](../features/audio-abr.md)** *(not yet implemented)* — when + implemented, composed in for multi-bitrate audio. Used as-is. +- **[`multi-language-audio`](../features/multi-language-audio.md)** *(partial — landed)* — + composed in unchanged; variant state exposes `userAudioTrackSelection` slot. + Variant engine composes `switchAudioTrack` (slot owner with filter + flush) + instead of `selectAudioTrack`; `setupAudioBufferActors` carries over from + the default engine. Consumer can write language filters or specific track + IDs for mid-stream switching within the audio-only variant. ## Customer-policy surface @@ -322,11 +325,13 @@ CDN entry: `packages/html/src/cdn/media/simple-hls-audio-only.ts` → Public re-export: `@videojs/react/media/simple-hls-audio-only`. -**Composed behaviors (Phase 1):** `syncPreload`, `trackLoadTriggers`, -`resolvePresentation`, `selectAudioTrack`, `resolveAudioTrack`, -`calculatePresentationDuration`, `setupMediaSource`, -`updateMediaSourceDuration`, `setupAudioBufferActors`, `trackCurrentTime`, -`loadAudioSegments`, `endOfStream`, `shareSignals`. +**Composed behaviors (current):** `syncPreload`, `trackLoadTriggers`, +`resolvePresentation`, `resolveAudioTrack`, `calculatePresentationDuration`, +`setupMediaSource`, `updateMediaSourceDuration`, `setupAudioBufferActors`, +`trackCurrentTime`, `switchAudioTrack`, `loadAudioSegments`, `endOfStream`, +`shareSignals`. (Phase 1 composed `selectAudioTrack`; `switchAudioTrack` +replaced it when [`multi-language-audio`](../features/multi-language-audio.md) +Tier 2 landed.) `endOfStream` composes unchanged from the default engine — it iterates buffer actors via `[videoBufferActor, audioBufferActor].filter(Boolean)` diff --git a/packages/spf/src/all.ts b/packages/spf/src/all.ts index c57df389..21b7fe74 100644 --- a/packages/spf/src/all.ts +++ b/packages/spf/src/all.ts @@ -98,8 +98,8 @@ export { resolvePresentation } from './playback/behaviors/resolve-presentation'; export { syncPreload } from './playback/behaviors/sync-preload'; // ============================================================================= -// Features (F9 - Quality Switching) +// Features — Track Switching (video ABR + audio language selection) // ============================================================================= -export type { QualitySwitchingConfig, QualitySwitchingState } from './playback/behaviors/quality-switching'; -export { DEFAULT_INITIAL_BANDWIDTH, switchVideoQuality } from './playback/behaviors/quality-switching'; +export type { TrackSwitchingConfig, TrackSwitchingState } from './playback/behaviors/track-switching'; +export { DEFAULT_INITIAL_BANDWIDTH, switchAudioTrack, switchVideoTrack } from './playback/behaviors/track-switching'; diff --git a/packages/spf/src/core/composition/create-composition.ts b/packages/spf/src/core/composition/create-composition.ts index 5d62c039..c8085461 100644 --- a/packages/spf/src/core/composition/create-composition.ts +++ b/packages/spf/src/core/composition/create-composition.ts @@ -262,7 +262,7 @@ export interface CompositionOptions` plus a - * runtime-supplied `currentTrack` for upgrade-vs-downgrade decisions. + * Selection context for `selectQuality`. The `bandwidth` field carries the + * current network estimate; `safetyMargin` / `upgradeMargin` override the + * defaults; `currentTrack` enables upgrade-vs-downgrade hysteresis. + * + * Shape matches the unified `selectOptimal` contract of + * `setupTrackSwitching` (`playback/behaviors/track-switching.ts`) so the + * function can be passed directly as a variant's `selectOptimal`. */ -export interface SelectQualityOpts extends Partial { +export interface SelectQualityCtx + extends Partial { + bandwidth: number; /** * Track currently selected. When supplied, `selectQuality` returns * `currentTrack` (no change) for upgrades that don't clear the * `upgradeMargin`. When omitted, no hysteresis is applied — the * computed optimal is returned regardless. */ - currentTrack?: PartiallyResolvedVideoTrack | VideoTrack; + currentTrack?: T; } /** - * Select the track to apply now, given current bandwidth, a current - * selection (optional), and tuning. Returns: + * Select the track to apply now, given a context that carries current + * bandwidth, an optional `currentTrack`, and tuning overrides. Returns: * * - The bandwidth-fitting optimal when no `currentTrack` is supplied. * - The optimal when it's a downgrade vs. `currentTrack` (downgrades @@ -67,28 +74,27 @@ export interface SelectQualityOpts extends Partial { * (stay put — caller checks identity to no-op). * * "Optimal" is the highest-bandwidth track where the available bandwidth - * meets the safety requirement (`currentBandwidth >= track.bandwidth / safetyMargin`). + * meets the safety requirement (`bandwidth >= track.bandwidth / safetyMargin`). * Falls back to the lowest-bandwidth track when nothing fits the safety * margin (preserves a definitive pick under under-bandwidth conditions). * * @example * const tracks = [low, mid, high]; - * selectQuality(tracks, 5_000_000, { currentTrack: low }); + * selectQuality(tracks, { bandwidth: 5_000_000, currentTrack: low }); * // Returns `high` if 5 Mbps clears safety AND high.bandwidth >= low.bandwidth * upgradeMargin. * // Returns `low` (no-op signal) otherwise. */ export function selectQuality( tracks: readonly (PartiallyResolvedVideoTrack | VideoTrack)[], - currentBandwidth: number, - opts: SelectQualityOpts = {} + ctx: SelectQualityCtx ): PartiallyResolvedVideoTrack | VideoTrack | undefined { if (tracks.length === 0) { return undefined; } - const safetyMargin = opts.safetyMargin ?? DEFAULT_QUALITY_CONFIG.safetyMargin; - const upgradeMargin = opts.upgradeMargin ?? DEFAULT_QUALITY_CONFIG.upgradeMargin; - const { currentTrack } = opts; + const safetyMargin = ctx.safetyMargin ?? DEFAULT_QUALITY_CONFIG.safetyMargin; + const upgradeMargin = ctx.upgradeMargin ?? DEFAULT_QUALITY_CONFIG.upgradeMargin; + const { bandwidth: currentBandwidth, currentTrack } = ctx; // Sort tracks by bandwidth (lowest first) const sortedTracks = tracks.slice().sort((a, b) => a.bandwidth - b.bandwidth); diff --git a/packages/spf/src/media/abr/tests/quality-selection.test.ts b/packages/spf/src/media/abr/tests/quality-selection.test.ts index 1baa331f..ed6caafd 100644 --- a/packages/spf/src/media/abr/tests/quality-selection.test.ts +++ b/packages/spf/src/media/abr/tests/quality-selection.test.ts @@ -25,7 +25,7 @@ describe('selectQuality', () => { ]; // With 2.5 Mbps, should select 720p (2 Mbps) - const selected = selectQuality(tracks, 2_500_000); + const selected = selectQuality(tracks, { bandwidth: 2_500_000 }); expect(selected?.id).toBe('720p'); }); @@ -38,7 +38,7 @@ describe('selectQuality', () => { ]; // With 200 Kbps, should select 360p - const selected = selectQuality(tracks, 200_000); + const selected = selectQuality(tracks, { bandwidth: 200_000 }); expect(selected?.id).toBe('360p'); }); @@ -52,7 +52,7 @@ describe('selectQuality', () => { ]; // With 10 Mbps, should select 1080p - const selected = selectQuality(tracks, 10_000_000); + const selected = selectQuality(tracks, { bandwidth: 10_000_000 }); expect(selected?.id).toBe('1080p'); }); @@ -64,11 +64,11 @@ describe('selectQuality', () => { // To select 1080p (4 Mbps), need 4M / 0.85 ≈ 4.7 Mbps // With 4.6 Mbps, should select 720p (15% safety margin) - const selected1 = selectQuality(tracks, 4_600_000); + const selected1 = selectQuality(tracks, { bandwidth: 4_600_000 }); expect(selected1?.id).toBe('720p'); // With 4.8 Mbps, should select 1080p - const selected2 = selectQuality(tracks, 4_800_000); + const selected2 = selectQuality(tracks, { bandwidth: 4_800_000 }); expect(selected2?.id).toBe('1080p'); }); @@ -76,7 +76,7 @@ describe('selectQuality', () => { const tracks: PartiallyResolvedVideoTrack[] = [createTrack('720p', 2_000_000), createTrack('1080p', 4_000_000)]; // Exactly 2M / 0.85 ≈ 2.35 Mbps required for 720p - const selected = selectQuality(tracks, 2_350_000); + const selected = selectQuality(tracks, { bandwidth: 2_350_000 }); expect(selected?.id).toBe('720p'); }); @@ -89,14 +89,14 @@ describe('selectQuality', () => { }; // With 0.9 margin, need 4M / 0.9 ≈ 4.4 Mbps - const selected = selectQuality(tracks, 4_500_000, config); + const selected = selectQuality(tracks, { bandwidth: 4_500_000, ...config }); expect(selected?.id).toBe('1080p'); }); }); describe('edge cases', () => { it('should return undefined for empty track list', () => { - const selected = selectQuality([], 5_000_000); + const selected = selectQuality([], { bandwidth: 5_000_000 }); expect(selected).toBeUndefined(); }); @@ -104,7 +104,7 @@ describe('selectQuality', () => { const tracks: PartiallyResolvedVideoTrack[] = [createTrack('720p', 2_000_000)]; // Should select the only available track - const selected = selectQuality(tracks, 1_000_000); + const selected = selectQuality(tracks, { bandwidth: 1_000_000 }); expect(selected?.id).toBe('720p'); }); @@ -115,7 +115,7 @@ describe('selectQuality', () => { ]; // Should prefer higher resolution at same bandwidth - const selected = selectQuality(tracks, 2_500_000); + const selected = selectQuality(tracks, { bandwidth: 2_500_000 }); expect(selected?.id).toBe('720p-high'); }); @@ -123,7 +123,7 @@ describe('selectQuality', () => { const tracks: PartiallyResolvedVideoTrack[] = [createTrack('360p', 500_000), createTrack('720p', 2_000_000)]; // Should select lowest quality - const selected = selectQuality(tracks, 0); + const selected = selectQuality(tracks, { bandwidth: 0 }); expect(selected?.id).toBe('360p'); }); @@ -151,7 +151,7 @@ describe('selectQuality', () => { const tracks = [track1, track2]; // Should handle missing width/height gracefully - const selected = selectQuality(tracks, 1_500_000); + const selected = selectQuality(tracks, { bandwidth: 1_500_000 }); expect(selected).toBeDefined(); }); }); @@ -166,7 +166,7 @@ describe('selectQuality', () => { ]; // With 2.5 Mbps, should still select 720p despite unsorted list - const selected = selectQuality(tracks, 2_500_000); + const selected = selectQuality(tracks, { bandwidth: 2_500_000 }); expect(selected?.id).toBe('720p'); }); @@ -178,7 +178,7 @@ describe('selectQuality', () => { createTrack('360p', 500_000), ]; - const selected = selectQuality(tracks, 1_500_000); + const selected = selectQuality(tracks, { bandwidth: 1_500_000 }); expect(selected?.id).toBe('480p'); }); }); @@ -194,19 +194,19 @@ describe('selectQuality', () => { it('should handle startup with low initial estimate', () => { // Startup: conservative 1 Mbps estimate - const selected = selectQuality(abrLadder, 1_000_000); + const selected = selectQuality(abrLadder, { bandwidth: 1_000_000 }); expect(selected?.id).toBe('360p'); // Stay conservative }); it('should handle steady state with good bandwidth', () => { // Good Wi-Fi: 5 Mbps - const selected = selectQuality(abrLadder, 5_000_000); + const selected = selectQuality(abrLadder, { bandwidth: 5_000_000 }); expect(selected?.id).toBe('720p'); // Upgrade threshold keeps it from 1080p }); it('should handle bandwidth drop gracefully', () => { // Bandwidth drops to 1.5 Mbps - const selected = selectQuality(abrLadder, 1_500_000); + const selected = selectQuality(abrLadder, { bandwidth: 1_500_000 }); expect(selected?.id).toBe('480p'); }); @@ -218,7 +218,7 @@ describe('selectQuality', () => { 6_000_000, // 1080p ]; - const selections = estimates.map((bw) => selectQuality(abrLadder, bw)); + const selections = estimates.map((bw) => selectQuality(abrLadder, { bandwidth: bw })); expect(selections[0]?.id).toBe('360p'); expect(selections[1]?.id).toBe('480p'); @@ -234,7 +234,7 @@ describe('selectQuality', () => { createTrack('1080p-low', 2_000_000, 1920, 1080), ]; - const selected = selectQuality(tracks, 2_500_000); + const selected = selectQuality(tracks, { bandwidth: 2_500_000 }); expect(selected?.id).toBe('1080p-low'); }); @@ -244,7 +244,7 @@ describe('selectQuality', () => { createTrack('tall', 2_000_000, 1280, 1440), // 1,843,200 pixels ]; - const selected = selectQuality(tracks, 2_500_000); + const selected = selectQuality(tracks, { bandwidth: 2_500_000 }); expect(selected?.id).toBe('tall'); // More total pixels }); }); @@ -256,11 +256,11 @@ describe('selectQuality', () => { // To select 1080p: need 4M / 0.85 ≈ 4.7 Mbps (15% headroom) // At 4.5 Mbps: below safety threshold for 1080p, select 720p - const selected1 = selectQuality(tracks, 4_500_000); + const selected1 = selectQuality(tracks, { bandwidth: 4_500_000 }); expect(selected1?.id).toBe('720p'); // At 4.8 Mbps: above safety threshold for 1080p, select 1080p - const selected2 = selectQuality(tracks, 4_800_000); + const selected2 = selectQuality(tracks, { bandwidth: 4_800_000 }); expect(selected2?.id).toBe('1080p'); }); @@ -272,7 +272,7 @@ describe('selectQuality', () => { ]; // With 2.4 Mbps: can fit 720p (needs 2M/0.85 = 2.35M) - const selected = selectQuality(tracks, 2_400_000); + const selected = selectQuality(tracks, { bandwidth: 2_400_000 }); expect(selected?.id).toBe('720p'); }); }); diff --git a/packages/spf/src/media/primitives/select-tracks.ts b/packages/spf/src/media/primitives/select-tracks.ts index a6c28932..148d335d 100644 --- a/packages/spf/src/media/primitives/select-tracks.ts +++ b/packages/spf/src/media/primitives/select-tracks.ts @@ -1,11 +1,5 @@ import { DEFAULT_QUALITY_CONFIG, selectQuality } from '../abr/quality-selection'; -import type { - AudioSelectionSet, - MaybeResolvedPresentation, - Presentation, - TrackType, - VideoSelectionSet, -} from '../types'; +import type { AudioSelectionSet, MaybeResolvedPresentation, TrackType, VideoSelectionSet } from '../types'; import { SelectedTrackIdKeyByType } from '../utils/track-selection'; /** @@ -36,19 +30,10 @@ export type TrackSelectionContext = Record; */ export type TrackSelectionAction = { type: 'presentation-loaded' }; -/** - * Base configuration for track selection. - * Generic over track type with discriminant `type` field. - */ -export interface TrackSelectionConfig { - type: T; -} - /** * Configuration for video track selection. - * Generic with default to 'video' for convenience. */ -export interface VideoSelectionConfig extends TrackSelectionConfig { +export interface VideoSelectionConfig { /** * Initial bandwidth estimate for cold start (bits per second). * Used to select video quality before we have real measurements. @@ -65,9 +50,8 @@ export interface VideoSelectionConfig extends Tra /** * Configuration for audio track selection. - * Generic with default to 'audio' for convenience. */ -export interface AudioSelectionConfig extends TrackSelectionConfig { +export interface AudioSelectionConfig { /** * Preferred audio language (ISO 639 code, e.g., "en", "es"). * If not specified, selects first audio track. @@ -77,11 +61,6 @@ export interface AudioSelectionConfig extends Tra /** * Configuration for text track selection. - * - * Standalone (not extending `TrackSelectionConfig<'text'>`) because - * `pickTextTrack` hardcodes the `'text'` type lookup internally — the - * discriminant field carries no information at runtime, so we don't - * require callers to pass it. */ export interface TextSelectionConfig { /** @@ -121,7 +100,7 @@ export interface TextSelectionConfig { * select, or `undefined` to leave the slot unset. * * Behaviors that own a track-selection slot (`selectAudioTrack`, - * `selectTextTrack`, `selectVideoTrack`, `switchVideoQuality`) accept a + * `selectTextTrack`, `selectVideoTrack`, `switchVideoTrack`) accept a * `TrackPicker` via config. The behavior passes its own config straight * through as the picker's second argument — pickers that need richer * options (language preferences, default-track filtering, bandwidth-aware @@ -156,8 +135,11 @@ export function pickFirstTrackId(presentation: MaybeResolvedPresentation, type: * @param config - Selection configuration (bandwidth, safety margin) * @returns Selected video track ID, or undefined if no video tracks */ -export function pickVideoTrack(presentation: Presentation, config: VideoSelectionConfig): string | undefined { - const videoSet = presentation.selectionSets.find((set) => set.type === 'video') as VideoSelectionSet | undefined; +export function pickVideoTrack( + presentation: MaybeResolvedPresentation, + config?: VideoSelectionConfig +): string | undefined { + const videoSet = presentation.selectionSets?.find((set) => set.type === 'video') as VideoSelectionSet | undefined; if (!videoSet || videoSet.switchingSets.length === 0) { return undefined; @@ -169,11 +151,11 @@ export function pickVideoTrack(presentation: Presentation, config: VideoSelectio return undefined; } - const initialBandwidth = config.initialBandwidth ?? DEFAULT_INITIAL_BANDWIDTH; - const safetyMargin = config.safetyMargin ?? DEFAULT_QUALITY_CONFIG.safetyMargin; + const initialBandwidth = config?.initialBandwidth ?? DEFAULT_INITIAL_BANDWIDTH; + const safetyMargin = config?.safetyMargin ?? DEFAULT_QUALITY_CONFIG.safetyMargin; // selectQuality works with both partially resolved and resolved tracks - const selected = selectQuality(switchingSet.tracks as any, initialBandwidth, { safetyMargin }); + const selected = selectQuality(switchingSet.tracks as any, { bandwidth: initialBandwidth, safetyMargin }); return selected?.id; } @@ -218,8 +200,11 @@ export function pickMaxResolutionVideoTrack(presentation: MaybeResolvedPresentat * @param config - Selection configuration (preferred language) * @returns Selected audio track ID, or undefined if no audio tracks */ -export function pickAudioTrack(presentation: Presentation, config: AudioSelectionConfig): string | undefined { - const audioSet = presentation.selectionSets.find((set) => set.type === 'audio') as AudioSelectionSet | undefined; +export function pickAudioTrack( + presentation: MaybeResolvedPresentation, + config?: AudioSelectionConfig +): string | undefined { + const audioSet = presentation.selectionSets?.find((set) => set.type === 'audio') as AudioSelectionSet | undefined; if (!audioSet || audioSet.switchingSets.length === 0) { return undefined; @@ -234,7 +219,7 @@ export function pickAudioTrack(presentation: Presentation, config: AudioSelectio const tracks = switchingSet.tracks; // Try preferred language first - if (config.preferredAudioLanguage) { + if (config?.preferredAudioLanguage) { const languageMatch = tracks.find((track) => track.language === config.preferredAudioLanguage); if (languageMatch) { return languageMatch.id; @@ -306,12 +291,8 @@ export function pickTextTrack( * * Generic over track type - works for video, audio, or text. */ -export function canSelectTrack( - state: TrackSelectionState, - config: TrackSelectionConfig -): boolean { - return !!state?.presentation?.selectionSets?.find(({ type }) => type === config.type)?.switchingSets?.[0]?.tracks - .length; +export function canSelectTrack(state: TrackSelectionState, type: TrackType): boolean { + return !!state?.presentation?.selectionSets?.find((set) => set.type === type)?.switchingSets?.[0]?.tracks.length; } /** @@ -325,9 +306,6 @@ export function canSelectTrack( * @TODO figure out reactive model for ABR cases - right now we're only selecting * if we have nothing selected (CJP) */ -export function shouldSelectTrack( - state: TrackSelectionState, - config: TrackSelectionConfig -): boolean { - return !state[SelectedTrackIdKeyByType[config.type]]; +export function shouldSelectTrack(state: TrackSelectionState, type: TrackType): boolean { + return !state[SelectedTrackIdKeyByType[type]]; } diff --git a/packages/spf/src/media/primitives/tests/select-tracks.test.ts b/packages/spf/src/media/primitives/tests/select-tracks.test.ts index c0c16c6f..2acadfa6 100644 --- a/packages/spf/src/media/primitives/tests/select-tracks.test.ts +++ b/packages/spf/src/media/primitives/tests/select-tracks.test.ts @@ -105,22 +105,22 @@ describe('pickVideoTrack', () => { const presentation = createPresentation({ video: tracks }); // With default 1 Mbps, should select 360p (500k fits with margin) - const selected = pickVideoTrack(presentation, { type: 'video' }); + const selected = pickVideoTrack(presentation); expect(selected).toBe('360p'); // With 3 Mbps, should select 720p (2M fits, 4M doesn't with 0.85 margin) - const selected2 = pickVideoTrack(presentation, { initialBandwidth: 3_000_000, type: 'video' }); + const selected2 = pickVideoTrack(presentation, { initialBandwidth: 3_000_000 }); expect(selected2).toBe('720p'); // With 5 Mbps, should select 1080p (4M fits with margin) - const selected3 = pickVideoTrack(presentation, { initialBandwidth: 5_000_000, type: 'video' }); + const selected3 = pickVideoTrack(presentation, { initialBandwidth: 5_000_000 }); expect(selected3).toBe('1080p'); }); it('returns undefined when no video tracks', () => { const presentation = createPresentation({ audio: [] }); - const selected = pickVideoTrack(presentation, { type: 'video' }); + const selected = pickVideoTrack(presentation); expect(selected).toBeUndefined(); }); @@ -147,7 +147,7 @@ describe('pickVideoTrack', () => { const presentation = createPresentation({ video: tracks }); // With 100 kbps (very low), should fall back to lowest (720p) - const selected = pickVideoTrack(presentation, { initialBandwidth: 100_000, type: 'video' }); + const selected = pickVideoTrack(presentation, { initialBandwidth: 100_000 }); expect(selected).toBe('720p'); }); @@ -170,7 +170,6 @@ describe('pickVideoTrack', () => { const selected = pickVideoTrack(presentation, { initialBandwidth: 2_050_000, safetyMargin: 0.95, - type: 'video', }); expect(selected).toBe('720p'); // Falls back since it's the only/lowest option }); @@ -306,7 +305,7 @@ describe('pickAudioTrack', () => { const presentation = createPresentation({ audio: tracks }); - const selected = pickAudioTrack(presentation, { type: 'audio' }); + const selected = pickAudioTrack(presentation); expect(selected).toBe('audio-en'); }); @@ -342,7 +341,7 @@ describe('pickAudioTrack', () => { const presentation = createPresentation({ audio: tracks }); - const selected = pickAudioTrack(presentation, { type: 'audio', preferredAudioLanguage: 'es' }); + const selected = pickAudioTrack(presentation, { preferredAudioLanguage: 'es' }); expect(selected).toBe('audio-es'); }); @@ -377,7 +376,7 @@ describe('pickAudioTrack', () => { const presentation = createPresentation({ audio: tracks }); - const selected = pickAudioTrack(presentation, { type: 'audio' }); + const selected = pickAudioTrack(presentation); expect(selected).toBe('audio-es'); }); @@ -414,14 +413,14 @@ describe('pickAudioTrack', () => { const presentation = createPresentation({ audio: tracks }); - const selected = pickAudioTrack(presentation, { type: 'audio', preferredAudioLanguage: 'en' }); + const selected = pickAudioTrack(presentation, { preferredAudioLanguage: 'en' }); expect(selected).toBe('audio-en'); }); it('returns undefined when no audio tracks', () => { const presentation = createPresentation({ video: [] }); - const selected = pickAudioTrack(presentation, { type: 'audio' }); + const selected = pickAudioTrack(presentation); expect(selected).toBeUndefined(); }); }); diff --git a/packages/spf/src/playback/actors/dom/segment-loader.ts b/packages/spf/src/playback/actors/dom/segment-loader.ts index 5890222b..a69ffe1f 100644 --- a/packages/spf/src/playback/actors/dom/segment-loader.ts +++ b/packages/spf/src/playback/actors/dom/segment-loader.ts @@ -289,9 +289,19 @@ export function createSegmentLoaderActor( * @todo Rename alongside LoadTask (e.g. planOps). * * Case 1 — Removes: forward and back buffer flush points, segment-aligned. - * No flush on track switch: appending new content overwrites existing buffer - * ranges, and the actor's time-aligned deduplication keeps the segment model - * accurate as new segments arrive. + * ABR-style track switches (same content, different bitrate) do not flush: + * appending new content overwrites existing buffer ranges, and the actor's + * time-aligned deduplication keeps the segment model accurate as new + * segments arrive. + * + * Cross-rendition track switches (audio language change, text language + * change) do flush: the buffered content is semantically incompatible with + * the newly-selected track, so overwrite-on-append would leave stale + * content playing until each replacement segment lands. Today's predicate: + * `actorCtx.initTrackLanguage !== track.language` — fires for language + * changes, no-ops for video / same-language audio bitrate switches. + * Future stage: pluggable predicate / strategy at actor construction time + * for codec-change (5.1 surround) and other cross-rendition shapes. * * Case 2 — Init: schedule if not yet committed for this track. * @@ -307,23 +317,80 @@ export function createSegmentLoaderActor( const currentTime = range?.start ?? 0; const tasks: LoadTask[] = []; - // Case 1: Removes + // Cross-rendition switch check (mid-stream language change). Fires when + // (a) an init segment has already been committed for some track, + // (b) the newly-selected track is a different track, and + // (c) the languages differ. The buffered range from the current segment + // boundary forward is treated as stale (new track's same-timestamp + // segments will overwrite it via MSE append-at-same-timestamp); no + // explicit `remove` task is emitted for the cross-rendition range. + // Computed against the currently-buffered segments (stable reference; + // new track's playlist may not be resolved yet at first load). + const isCrossRenditionSwitch = + actorCtx.initTrackId !== undefined && + actorCtx.initTrackId !== track.id && + actorCtx.initTrackLanguage !== track.language; + + // Two categories of "buffered content that should not gate planning": + // + // - `removes` — content that needs an explicit `remove` task (out-of-window + // forward content, back-buffer content beyond the keep window). Emitted + // as `{ type: 'remove' }` tasks. + // - `staleRanges` — content that the new appends will overwrite at the + // same timestamps (cross-rendition switch). No explicit `remove` — + // MSE's overwrite-on-append handles it, which gives a much smaller + // perceived audio gap than `remove`-then-fetch-then-append. + // + // Both categories affect `effectiveBuffered` so `getSegmentsToLoad` + // re-plans new-track segments inside them. + const removes: Array<{ start: number; end: number }> = []; + const staleRanges: Array<{ start: number; end: number }> = []; if (range) { + if (isCrossRenditionSwitch) { + // Mark current-segment-start onward as stale. Falls back to the + // first buffered segment after currentTime when the playhead sits + // in a buffer gap. + const currentSeg = actorCtx.segments.find( + (s) => s.startTime <= currentTime && s.startTime + s.duration > currentTime + ); + const staleStart = currentSeg?.startTime ?? actorCtx.segments.find((s) => s.startTime > currentTime)?.startTime; + if (staleStart !== undefined) { + staleRanges.push({ start: staleStart, end: Infinity }); + } + } const forwardFlushStart = calculateForwardFlushPoint(bufferedSegments, currentTime, forwardBufferConfig); if (forwardFlushStart < Infinity) { - tasks.push({ type: 'remove', start: forwardFlushStart, end: Infinity }); + removes.push({ start: forwardFlushStart, end: Infinity }); } const backFlushEnd = calculateBackBufferFlushPoint(bufferedSegments, currentTime, backBufferConfig); if (backFlushEnd > 0) { - tasks.push({ type: 'remove', start: 0, end: backFlushEnd }); + removes.push({ start: 0, end: backFlushEnd }); } + for (const r of removes) tasks.push({ type: 'remove', start: r.start, end: r.end }); } + // Treat any segment overlapping a planned remove OR a stale range as + // not-buffered. Without this, sibling renditions that share segment IDs + // and startTimes (audio language variants) would see the new track's + // same-startTime segments marked "buffered" by the pre-flush snapshot, + // skip them in `getSegmentsToLoad`, and leave a permanent gap from + // `currentSegmentStart` to the end of the old buffer window — stalling + // playback. + const overlapsStale = (seg: { startTime: number; duration: number }): boolean => { + const segEnd = seg.startTime + seg.duration; + return ( + removes.some((r) => seg.startTime < r.end && segEnd > r.start) || + staleRanges.some((r) => seg.startTime < r.end && segEnd > r.start) + ); + }; + const effectiveBuffered = + removes.length + staleRanges.length > 0 ? bufferedSegments.filter((s) => !overlapsStale(s)) : bufferedSegments; + // Case 2: Init if (actorCtx.initTrackId !== track.id) { tasks.push({ type: 'append-init', - meta: { trackId: track.id }, + meta: { trackId: track.id, language: track.language }, url: track.initialization.url, ...(track.initialization.byteRange !== undefined && { byteRange: track.initialization.byteRange }), }); @@ -333,14 +400,19 @@ export function createSegmentLoaderActor( if (range) { const segmentsToLoad = getSegmentsToLoad( track.segments, - bufferedSegments, + effectiveBuffered, currentTime, forwardBufferConfig ).filter((seg) => { // Quality-aware filter: skip segments already covered by equal-or-higher-quality // content in the actor context. Preserves buffered high-quality content during // ABR downgrades; loads during upgrades and for uncovered positions. - const existing = actorCtx.segments.find((s) => Math.abs(s.startTime - seg.startTime) < SEGMENT_TIME_EPSILON); + // Actor entries that overlap a planned remove OR a stale range are treated + // as nonexistent here — they're about to be flushed or overwritten, so the + // new-track segment must load regardless of the existing entry's bandwidth. + const existing = actorCtx.segments.find( + (s) => !overlapsStale(s) && Math.abs(s.startTime - seg.startTime) < SEGMENT_TIME_EPSILON + ); // Partial segments are still streaming — treat as not buffered so they // are always re-planned (avoids relying on incomplete data). if (existing?.partial) return true; diff --git a/packages/spf/src/playback/actors/dom/source-buffer.ts b/packages/spf/src/playback/actors/dom/source-buffer.ts index 90b7d318..59dcd976 100644 --- a/packages/spf/src/playback/actors/dom/source-buffer.ts +++ b/packages/spf/src/playback/actors/dom/source-buffer.ts @@ -21,7 +21,18 @@ export type AppendSegmentMeta = Pick & export type { AppendData }; -export type AppendInitMessage = { type: 'append-init'; data: AppendData; meta: { trackId: Track['id'] } }; +export type AppendInitMessage = { + type: 'append-init'; + data: AppendData; + /** + * `language` is captured alongside `trackId` so downstream loaders can + * compare the buffered track's language to the newly-selected track's + * language and decide whether ahead-buffer flush is warranted on track + * switch (see `segment-loader`'s `planTasks`). Undefined for video and + * for audio without explicit `LANGUAGE` attribute. + */ + meta: { trackId: Track['id']; language?: string }; +}; export type AppendSegmentMessage = { type: 'append-segment'; data: AppendData; meta: AppendSegmentMeta }; export type RemoveMessage = { type: 'remove'; start: number; end: number }; export type IndividualSourceBufferMessage = AppendInitMessage | AppendSegmentMessage | RemoveMessage; @@ -37,6 +48,13 @@ export type SourceBufferActorState = 'idle' | 'updating' | 'destroyed'; /** Non-finite (extended) data managed by the actor — the XState "context". */ export interface SourceBufferActorContext { initTrackId?: string | undefined; + /** + * Language of the most recently appended init segment's track (when + * present on the playlist). Used by the segment-loader's `planTasks` + * to detect cross-language switches and schedule ahead-buffer flush. + * Undefined for video and for language-less audio. + */ + initTrackLanguage?: string | undefined; segments: Array< Pick & { trackId: Track['id']; @@ -91,7 +109,7 @@ function appendInitTask( await appendSegment(sourceBuffer, message.data); // No abort check here: the physical SourceBuffer has been modified, so // the model must be updated to match regardless of signal state. - return { ...ctx, initTrackId: message.meta.trackId }; + return { ...ctx, initTrackId: message.meta.trackId, initTrackLanguage: message.meta.language }; }); } diff --git a/packages/spf/src/playback/actors/dom/tests/segment-loader.test.ts b/packages/spf/src/playback/actors/dom/tests/segment-loader.test.ts new file mode 100644 index 00000000..d485b2a6 --- /dev/null +++ b/packages/spf/src/playback/actors/dom/tests/segment-loader.test.ts @@ -0,0 +1,238 @@ +import { describe, expect, it, vi } from 'vitest'; +import { signal } from '../../../../core/signals/primitives'; +import type { AudioTrack } from '../../../../media/types'; +import { createSegmentLoaderActor } from '../segment-loader'; +import type { + SourceBufferActor, + SourceBufferActorContext, + SourceBufferActorState, + SourceBufferMessage, +} from '../source-buffer'; + +// --------------------------------------------------------------------------- +// Mock helpers +// --------------------------------------------------------------------------- + +interface MockSourceBufferActor { + snapshot: ReturnType>; + send: ReturnType; + destroy: ReturnType; +} + +function createMockBufferActor(ctx: Partial = {}): MockSourceBufferActor { + return { + snapshot: signal<{ value: SourceBufferActorState; context: SourceBufferActorContext }>({ + value: 'idle', // stay idle so waitForIdle resolves immediately + context: { segments: [], bufferedRanges: [], initTrackId: undefined, ...ctx }, + }), + send: vi.fn(), + destroy: vi.fn(), + }; +} + +function makeAudioTrack(id: string, overrides: Partial = {}): AudioTrack { + return { + type: 'audio', + id, + url: `http://example.com/${id}.m3u8`, + bandwidth: 128_000, + mimeType: 'audio/mp4', + codecs: ['mp4a.40.2'], + groupId: 'audio', + name: id, + sampleRate: 48000, + channels: 2, + startTime: 0, + duration: 30, + initialization: { url: `http://example.com/${id}-init.mp4` }, + segments: [ + { id: `${id}-0`, url: `http://example.com/${id}-0.m4s`, startTime: 0, duration: 6 }, + { id: `${id}-1`, url: `http://example.com/${id}-1.m4s`, startTime: 6, duration: 6 }, + { id: `${id}-2`, url: `http://example.com/${id}-2.m4s`, startTime: 12, duration: 6 }, + ], + ...overrides, + }; +} + +// Mock fetchBytes — returns an empty async iterable; segments effectively +// don't append. We only care about which messages the segment-loader +// dispatches to the source-buffer actor, particularly the `remove` for +// cross-rendition flush. +const mockFetchBytes = vi.fn(async () => { + return (async function* () {})(); +}); + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('createSegmentLoaderActor — planTasks cross-rendition switch', () => { + it('does NOT emit an explicit remove on cross-rendition switch; relies on MSE overwrite-on-append', async () => { + // Audio renditions share segment IDs and startTimes. Appending the new + // track's segments at the same timestamps overwrites the old data in + // the SourceBuffer — no explicit `remove` task needed, and no audible + // silence-gap during the swap. planTasks emits only the new-track init + // + new-track segments inside the (formerly-flushed) range. + const bufferActor = createMockBufferActor({ + initTrackId: 'audio-en', + initTrackLanguage: 'en', + segments: [ + { id: 'audio-en-0', startTime: 0, duration: 6, trackId: 'audio-en' }, + { id: 'audio-en-1', startTime: 6, duration: 6, trackId: 'audio-en' }, + { id: 'audio-en-2', startTime: 12, duration: 6, trackId: 'audio-en' }, + ], + bufferedRanges: [{ start: 0, end: 18 }], + }); + + const loader = createSegmentLoaderActor(bufferActor as unknown as SourceBufferActor, mockFetchBytes); + + const newTrack = makeAudioTrack('audio-es', { language: 'es' }); + // Playhead at 2s (mid-segment-0). currentSegmentStart = 0. + loader.send({ type: 'load', track: newTrack, range: { start: 2, end: 20 } }); + + // Wait for the loader to schedule its work (init + segments). + await vi.waitFor(() => { + const initCalls = bufferActor.send.mock.calls.filter((c) => (c[0] as SourceBufferMessage).type === 'append-init'); + expect(initCalls.length).toBeGreaterThan(0); + }); + + // No cross-rendition remove emitted. Forward / back flushes are + // computed independently and don't fire here (the buffered range is + // entirely inside the load window, and no back-buffer threshold is + // crossed at currentTime=2). + const removeMsgs = bufferActor.send.mock.calls + .map((c) => c[0] as SourceBufferMessage) + .filter((m): m is Extract => m.type === 'remove'); + + expect(removeMsgs).toHaveLength(0); + + loader.destroy(); + }); + + it('does not dispatch cross-rendition flush when languages match (audio-abr-style switch)', async () => { + // Same language, different track id (e.g., audio bitrate variant in + // same language group). planTasks should NOT emit a cross-rendition + // flush — appending new segments overwrites time-aligned ranges. + const bufferActor = createMockBufferActor({ + initTrackId: 'audio-en-128k', + initTrackLanguage: 'en', + segments: [{ id: 'audio-en-128k-0', startTime: 0, duration: 6, trackId: 'audio-en-128k' }], + bufferedRanges: [{ start: 0, end: 6 }], + }); + + const loader = createSegmentLoaderActor(bufferActor as unknown as SourceBufferActor, mockFetchBytes); + + const newTrack = makeAudioTrack('audio-en-256k', { language: 'en' }); + loader.send({ type: 'load', track: newTrack, range: { start: 2, end: 20 } }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const crossRenditionRemove = bufferActor.send.mock.calls + .map((c) => c[0] as SourceBufferMessage) + .find( + (m): m is Extract => + m.type === 'remove' && m.start > 0 && m.end === Infinity + ); + + // Note: a forward-buffer flush may fire with start === forwardFlushPoint + // and end === Infinity, but only when the buffer overflows the forward + // target. The cross-rendition flush has start === nextSegmentBoundary. + // Since languages match here, NO cross-rendition flush should fire. + // (Forward-buffer flush with a small forward-buffer is independent.) + expect(crossRenditionRemove?.start).not.toBe(6); + + loader.destroy(); + }); + + it('does not dispatch cross-rendition flush on initial load (no prior initTrackId)', async () => { + // No buffered track yet (initTrackId undefined). First load is the + // initial setup — should emit append-init but no cross-rendition flush. + const bufferActor = createMockBufferActor(); + const loader = createSegmentLoaderActor(bufferActor as unknown as SourceBufferActor, mockFetchBytes); + + const track = makeAudioTrack('audio-en', { language: 'en' }); + loader.send({ type: 'load', track, range: { start: 0, end: 20 } }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const removeCalls = bufferActor.send.mock.calls.filter((c) => (c[0] as SourceBufferMessage).type === 'remove'); + expect(removeCalls).toHaveLength(0); + + loader.destroy(); + }); + + it('schedules new-track segments inside the cross-rendition stale range (including current segment)', async () => { + // Cross-rendition audio renditions share segment IDs and startTimes + // (e.g., every variant has its own `segment-1` at startTime=6). + // planTasks treats the range from `currentSegmentStart` forward as + // stale (will be overwritten by new appends via MSE + // overwrite-on-append) and re-schedules every new-track segment in + // that range — including the segment containing the playhead. The + // pre-flush `bufferedSegments` snapshot would otherwise mark those + // same-startTime new-track segments "already buffered" and skip them. + const bufferActor = createMockBufferActor({ + initTrackId: 'audio-en', + initTrackLanguage: 'en', + // Buffered seg-0 (0-6), seg-1 (6-12), seg-2 (12-18). + segments: [ + { id: 'audio-en-0', startTime: 0, duration: 6, trackId: 'audio-en' }, + { id: 'audio-en-1', startTime: 6, duration: 6, trackId: 'audio-en' }, + { id: 'audio-en-2', startTime: 12, duration: 6, trackId: 'audio-en' }, + ], + bufferedRanges: [{ start: 0, end: 18 }], + }); + + const loader = createSegmentLoaderActor(bufferActor as unknown as SourceBufferActor, mockFetchBytes); + + const newTrack = makeAudioTrack('audio-es', { language: 'es' }); + // Playhead at 2s (mid-seg-0). currentSegmentStart = 0 → stale range + // is [0, Infinity). All three segments overlap and must be re-scheduled. + loader.send({ type: 'load', track: newTrack, range: { start: 2, end: 20 } }); + + await vi.waitFor(() => { + const appendSegmentCalls = bufferActor.send.mock.calls.filter( + (c) => (c[0] as SourceBufferMessage).type === 'append-segment' + ); + expect(appendSegmentCalls.length).toBeGreaterThan(0); + }); + + const segmentStartTimes = bufferActor.send.mock.calls + .map((c) => c[0] as SourceBufferMessage) + .filter((m): m is Extract => m.type === 'append-segment') + .map((m) => m.meta.startTime); + + // The current segment (startTime 0, containing playhead at 2) is + // re-scheduled so the audible switch happens within milliseconds + // of the new-track segment landing rather than waiting for the next + // boundary. + expect(segmentStartTimes).toContain(0); + expect(segmentStartTimes).toContain(6); + expect(segmentStartTimes).toContain(12); + + loader.destroy(); + }); + + it('captures language into append-init meta for downstream tracking', async () => { + // Verify that planTasks includes `language` in the append-init meta + // so the SourceBufferActor can capture initTrackLanguage on commit. + const bufferActor = createMockBufferActor(); + const loader = createSegmentLoaderActor(bufferActor as unknown as SourceBufferActor, mockFetchBytes); + + const track = makeAudioTrack('audio-es', { language: 'es' }); + loader.send({ type: 'load', track, range: { start: 0, end: 20 } }); + + await vi.waitFor(() => { + const initCall = bufferActor.send.mock.calls.find((c) => (c[0] as SourceBufferMessage).type === 'append-init'); + expect(initCall).toBeDefined(); + }); + + const initMsg = bufferActor.send.mock.calls + .map((c) => c[0] as SourceBufferMessage) + .find((m): m is Extract => m.type === 'append-init')!; + + expect(initMsg.meta.trackId).toBe('audio-es'); + expect(initMsg.meta.language).toBe('es'); + + loader.destroy(); + }); +}); diff --git a/packages/spf/src/playback/actors/dom/tests/source-buffer.test.ts b/packages/spf/src/playback/actors/dom/tests/source-buffer.test.ts index 5c276973..647b7943 100644 --- a/packages/spf/src/playback/actors/dom/tests/source-buffer.test.ts +++ b/packages/spf/src/playback/actors/dom/tests/source-buffer.test.ts @@ -265,6 +265,37 @@ describe('createSourceBufferActor', () => { actor.destroy(); }); + it('captures initTrackLanguage from append-init meta for downstream cross-rendition flush detection', async () => { + // Multi-language-audio Tier 2 prerequisite: the segment-loader's + // planTasks compares `actorCtx.initTrackLanguage` against the + // newly-selected track's language to decide whether to flush the + // ahead-buffer on track switch. The actor captures language alongside + // trackId from the append-init meta. + const sourceBuffer = makeSourceBuffer(); + const actor = createSourceBufferActor(sourceBuffer); + + actor.send({ type: 'append-init', data: new ArrayBuffer(4), meta: { trackId: 'audio-en', language: 'en' } }); + await vi.waitFor(() => expect(actor.snapshot.get().value).toBe('idle')); + + expect(actor.snapshot.get().context.initTrackId).toBe('audio-en'); + expect(actor.snapshot.get().context.initTrackLanguage).toBe('en'); + + actor.destroy(); + }); + + it('leaves initTrackLanguage undefined when append-init meta omits language (video)', async () => { + const sourceBuffer = makeSourceBuffer(); + const actor = createSourceBufferActor(sourceBuffer); + + actor.send({ type: 'append-init', data: new ArrayBuffer(4), meta: { trackId: 'video-1' } }); + await vi.waitFor(() => expect(actor.snapshot.get().value).toBe('idle')); + + expect(actor.snapshot.get().context.initTrackId).toBe('video-1'); + expect(actor.snapshot.get().context.initTrackLanguage).toBeUndefined(); + + actor.destroy(); + }); + // --------------------------------------------------------------------------- // append-segment // --------------------------------------------------------------------------- diff --git a/packages/spf/src/playback/behaviors/calculate-presentation-duration.ts b/packages/spf/src/playback/behaviors/calculate-presentation-duration.ts index b0f73ca6..93ca102c 100644 --- a/packages/spf/src/playback/behaviors/calculate-presentation-duration.ts +++ b/packages/spf/src/playback/behaviors/calculate-presentation-duration.ts @@ -65,7 +65,7 @@ function calculatePresentationDurationSetup({ presentation: Signal; // selectedVideoTrackId / selectedAudioTrackId are read defensively — // they are *not* declared in this behavior's stateKeys. The slots are - // contributed by other behaviors (`switchVideoQuality` writes the + // contributed by other behaviors (`switchVideoTrack` writes the // video slot in the default engine; `selectAudioTrack` writes the // audio slot) which compose conditionally per engine variant. Treating // each signal as optional lets calculatePresentationDuration stay diff --git a/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts b/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts index 2647d723..3bcfdfa5 100644 --- a/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts +++ b/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts @@ -28,7 +28,7 @@ * with its upstream and downstream siblings: * * 1. **Upstream — default selections land in one `runPending`.** - * `selectAudioTrack` (default audio) and `switchVideoQuality` + * `selectAudioTrack` (default audio) and `switchVideoTrack` * (default video) both subscribe to `state.presentation` flipping to * resolved; their effects run in the same `runPending` iteration and * write `selectedAudioTrackId` + `selectedVideoTrackId` within it. @@ -256,6 +256,13 @@ export const setupVideoBufferActors = defineBehavior({ * `internal/design/spf/features/audio-abr.md` for the design surface * (bandwidth-state sharing, multi-writer coordination, EWMA mixed- * source sampling). + * + * **Mid-stream audio track switching is NOT this behavior's concern.** + * Slot writes to `selectedAudioTrackId` (default selection, programmatic + * filter-driven, future ABR) are owned by `switchAudioTrack` / future + * `switchAudioQuality` in `track-switching.ts`. Flush orchestration on + * track change is dispatched from there via `audioBufferActor.send(...)` + * — keeping this setup behavior focused on per-source actor lifecycle. */ export const setupAudioBufferActors = defineBehavior({ stateKeys: ['presentation', 'selectedAudioTrackId'] as const, diff --git a/packages/spf/src/playback/behaviors/dom/sync-text-tracks.ts b/packages/spf/src/playback/behaviors/dom/sync-text-tracks.ts index 351351ee..39969e47 100644 --- a/packages/spf/src/playback/behaviors/dom/sync-text-tracks.ts +++ b/packages/spf/src/playback/behaviors/dom/sync-text-tracks.ts @@ -118,7 +118,7 @@ function syncTextTracksSetup({ // `getTracksByType('text', ...)` returns text tracks only — the // selection-set filter inside the helper ensures that — but its // declared return is the wide track union. Mirror the cast - // pattern used by `quality-switching` for the video branch. + // pattern used by `track-switching` for the video branch. const modelTextTracks = getTracksByType(state.presentation.get()!, 'text') as readonly ( | PartiallyResolvedTextTrack | TextTrack diff --git a/packages/spf/src/playback/behaviors/quality-switching.ts b/packages/spf/src/playback/behaviors/quality-switching.ts deleted file mode 100644 index 231c23e0..00000000 --- a/packages/spf/src/playback/behaviors/quality-switching.ts +++ /dev/null @@ -1,342 +0,0 @@ -/** - * **Manage the per-type ABR-eligible track selection slot.** While a - * presentation is resolved, owns the slot's lifecycle: pick a default, - * dynamically adjust based on media metrics (today: bandwidth via - * `selectQuality`), and clear on src unload. User intent is expressed as a - * partial-track description in a sibling slot (e.g. - * `userVideoTrackSelection`) which constrains the candidate set for - * selection — when the constraint narrows candidates to exactly one, the - * choice is fully determined and ABR is short-circuited (no bandwidth read, - * no effect re-fire on bandwidth changes). - * - * Lifecycle: `'presentation-unresolved'` ↔ `'presentation-resolved'`. The - * `'presentation-resolved'` state owns the selection slot; its - * entry-returned cleanup clears the slot on exit (canonical - * cleanup-binds-to-setup per `reactors.md`). - * - * Hysteresis: downgrades apply immediately; upgrades require the optimal - * track's bandwidth to exceed the current track's by `upgradeMargin`. No - * temporal state — short-term smoothing is the bandwidth estimator's job. - * - * Initial pick is configurable via `config.picker` — pass any `TrackPicker` - * to override the bandwidth-aware default for the empty-slot case. ABR - * re-evaluation (downgrade/upgrade) is unaffected. Composing - * `switchVideoQuality` with `picker: (p) => pickFirstTrackId(p, 'video')` - * yields "first track at load, ABR adjusts from there" — a non-bandwidth - * initial pick paired with bandwidth-driven re-evaluation. - */ - -import { defineBehavior } from '../../core/composition/create-composition'; -import { createMachineReactor } from '../../core/reactors/create-machine-reactor'; -import { computed, peek, type ReadonlySignal, type Signal } from '../../core/signals/primitives'; -import { - DEFAULT_QUALITY_CONFIG, - type QualityConfig, - selectLowestQuality, - selectQuality, -} from '../../media/abr/quality-selection'; -import type { TrackPicker } from '../../media/primitives/select-tracks'; -import { - isResolvedPresentation, - type MaybeResolvedPresentation, - type PartiallyResolvedVideoTrack, - type VideoTrack, -} from '../../media/types'; -import { getTracksByType } from '../../media/utils/tracks'; -import type { BandwidthConfig, BandwidthState } from '../../network/bandwidth-estimator'; -import { DEFAULT_BANDWIDTH_CONFIG, getBandwidthEstimate } from '../../network/bandwidth-estimator'; - -export interface QualitySwitchingState { - presentation?: MaybeResolvedPresentation; - bandwidthState?: BandwidthState; - selectedVideoTrackId?: string; - /** - * Partial-track description expressing user intent. When set, narrows - * the ABR candidate set to tracks matching every present field. The - * common case is `{ id: 'specific-track-id' }` for a "manual quality" - * pick, but other partial shapes work — e.g., `{ height: 720 }` would - * constrain to 720p tracks; ABR continues to pick among them. - * - * When the narrowed candidates contain exactly one track, ABR is - * short-circuited entirely (no bandwidth read, no effect re-fire). - * - * When the filter matches no tracks in the current presentation (e.g., - * user-picked id from a previous source doesn't exist here), falls back - * to the unfiltered set rather than stalling playback. - */ - userVideoTrackSelection?: Partial; -} - -export interface QualitySwitchingConfig { - /** - * Quality-selection tuning. `safetyMargin` is the bandwidth-headroom - * multiplier; `upgradeMargin` is the hysteresis ratio gating upgrades. - * Defaults: `DEFAULT_QUALITY_CONFIG` (0.85 / 1.15). - */ - quality?: Partial; - - /** - * Bandwidth-estimator tuning passed through to `getBandwidthEstimate`. - * Merged over `DEFAULT_BANDWIDTH_CONFIG`. - */ - bandwidth?: Partial; - - /** - * Bandwidth estimate in bps to use before enough samples have been collected. - * Default: 5_000_000 (5 Mbps). - */ - initialBandwidth?: number; - - /** - * Override the initial-pick algorithm. When set, the picker is called the - * first time the slot is empty in the `'presentation-resolved'` state; - * its returned id is set verbatim (no bandwidth-aware logic). Subsequent - * ABR re-evaluation (downgrade/upgrade by bandwidth + hysteresis) is - * unaffected and runs as usual. - * - * Default (no picker): a bandwidth-aware initial pick driven by - * `initialBandwidth` and `quality.safetyMargin`, identical to ABR's - * downgrade branch. - * - * Honors of `userVideoTrackSelection` are the picker's responsibility - * when overridden. If the picker returns `undefined`, the bandwidth-aware - * default pick fires (graceful fallback). - */ - picker?: TrackPicker; -} - -/** Default initial-bandwidth value used by quality-switching variants before measurements arrive. */ -export const DEFAULT_INITIAL_BANDWIDTH = 5_000_000; - -// ============================================================================ -// Specialization helper -// -// `setupQualitySwitching` has the same shape as a Behavior `setup` function: -// `({ state, config }) => Reactor`. Each `switchXQuality` export below calls -// it from inside its own `defineBehavior` setup, passing the per-type slot -// keys, track type, and selection algorithm explicitly via three generic -// parameters — `S` (selection slot key), `U` (user-selection slot key), `T` -// (ABR track type). The per-type export is the single point that ties them -// together; the helper stays slot-agnostic. -// -// -- Design note: why narrow `SelectionKey` / `UserSelectionKey` unions ---- -// Goal we did not reach: have callers "fully pass in" the slot keys, with -// the helper enforcing zero internal knowledge of which literals are valid. -// What blocks it: indexing a mapped-type intersection by a generic key. -// When `S extends keyof QualitySwitchingState` (or `string`), TS conservatively -// treats `state[selectionKey]` as the union of every possible match across -// the intersected mapped portions — including the fixed-key signals -// (`presentation`, `bandwidthState`) — and widens to their value-type union. -// The sibling pattern hits the same constraint and answers it the same way: -// `SelectedTrackKey` in `select-tracks.ts:43` is a hardcoded narrow union -// for the same reason. -// -// Routes considered for "fully passed in", with trade-offs (left here for -// the larger-group conversation): -// -// A. Derived constraint — `Exclude`. Removes the literal enumeration; same narrowness, -// computed from the state shape. Still "knows" the fixed-key names. -// B. Broad constraint (`S extends keyof QualitySwitchingState`) + access-site -// casts (`state[selectionKey] as Signal` at the top -// of the helper). Caller passes any keys; the type system stops checking -// S ≠ U or that the keys actually refer to selection slots. Once you -// cast at the top, the body is structurally the original remap pattern -// with extra ceremony. -// C. Remap pattern (pre-refactor) — caller passes signals under logical -// names (`selection`, `userSelection`); helper has no key generics. -// Cleanest types; inconsistent with the sibling helpers. -// -// Current pick is the narrow-union route because it matches siblings and -// the unions read as documentation ("these are the slots this helper -// manages") rather than restriction. Extending for audio is one literal -// per union. -// -------------------------------------------------------------------------- -// ============================================================================ - -type AbrTrack = { id: string; bandwidth: number }; - -type SelectionKey = 'selectedVideoTrackId'; -type UserSelectionKey = 'userVideoTrackSelection'; - -// Each mapped value references `P` so TS keeps the per-key dependency and -// resolves `state[selectionKey]` / `state[userSelectionKey]` to the right -// arm. `T` (track type) deliberately stays out of the state map — pulling -// it in detaches the user-selection mapped value from `P` and TS collapses -// the intersection. T flows through `QualitySwitchingSetupConfig` instead; -// the user-filter access casts at the read site (see below). -type QualitySwitchingStateMap = { - presentation: ReadonlySignal; - bandwidthState: ReadonlySignal; -} & { [P in S]: Signal } & { [P in U]: ReadonlySignal }; - -interface QualitySwitchingSetupConfig - extends QualitySwitchingConfig { - selectionKey: S; - userSelectionKey: U; - getTracks: (presentation: MaybeResolvedPresentation) => readonly T[]; - selectOptimal: ( - tracks: readonly T[], - bandwidth: number, - opts: { safetyMargin: number; upgradeMargin: number; currentTrack?: T } - ) => T | undefined; -} - -function setupQualitySwitching({ - state, - config, -}: { - state: QualitySwitchingStateMap; - config: QualitySwitchingSetupConfig; -}) { - const safetyMargin = config.quality?.safetyMargin ?? DEFAULT_QUALITY_CONFIG.safetyMargin; - const upgradeMargin = config.quality?.upgradeMargin ?? DEFAULT_QUALITY_CONFIG.upgradeMargin; - const initialBandwidth = config.initialBandwidth ?? DEFAULT_INITIAL_BANDWIDTH; - const bandwidthConfig: BandwidthConfig = { ...DEFAULT_BANDWIDTH_CONFIG, ...config.bandwidth }; - const { selectionKey, userSelectionKey, getTracks, selectOptimal } = config; - - const derivedStateSignal = computed(() => - isResolvedPresentation(state.presentation.get()) - ? ('presentation-resolved' as const) - : ('presentation-unresolved' as const) - ); - - return createMachineReactor({ - initial: 'presentation-unresolved', - monitor: () => derivedStateSignal.get(), - states: { - 'presentation-unresolved': {}, - 'presentation-resolved': { - // Canonical cleanup-binds-to-setup: the selection slot's valid - // lifespan is exactly 'presentation-resolved'. Clear fires on - // 'presentation-resolved' exit, covering both src unload and - // behavior destroy. - entry: () => () => state[selectionKey].set(undefined), - effects: [ - () => { - const presentation = peek(state.presentation); - if (!presentation) return; - - const allTracks = getTracks(presentation); - const [firstAllTrack] = allTracks; - if (!firstAllTrack) return; - - // State stores the filter as `Partial` (user-facing - // shape — includes V-only fields like `height`); the helper - // works against `Partial` so filter and track access share - // one index type. Filter keys absent on a partially-resolved - // track read as `undefined` and just exclude that track. - const userFilter = state[userSelectionKey].get() as Partial | undefined; - const matching = userFilter - ? allTracks.filter((track) => { - for (const key in userFilter) { - const filterValue = userFilter[key as keyof T]; - if (filterValue !== undefined && track[key as keyof T] !== filterValue) return false; - } - return true; - }) - : allTracks; - // Fall back to all tracks when the filter excludes everything - // (e.g., user-picked id doesn't exist in the current source). - const candidates = matching.length > 0 ? matching : allTracks; - if (!candidates.length) return; - - const selectedId = state[selectionKey].get(); - - // Common case: user has fully constrained the choice (e.g., - // `{ id: 'specific-720p' }` narrows to a single track). Skip - // the ABR path — and crucially, don't read `bandwidthState` - // so the effect doesn't re-fire on bandwidth changes while - // the user's selection holds. - if (candidates.length === 1) { - if (candidates[0]!.id !== selectedId) state[selectionKey].set(candidates[0]!.id); - return; - } - - // Read bandwidth up front to establish the signal subscription — - // future bandwidth changes must re-fire this effect even when - // the picker branch below takes the early-return path. (If the - // picker bails out without ever touching `bandwidthState`, ABR - // would otherwise be deaf to subsequent bandwidth changes.) - // - // Single path for pre-trust and post-trust: - // `getBandwidthEstimate` returns `initialBandwidth` when state - // is undefined or bytes sampled hasn't crossed `minTotalBytes`, - // so the initial pick + early-ABR window run the same - // `selectOptimal` path as a fully-trusted measurement. - const bandwidth = getBandwidthEstimate(state.bandwidthState.get(), initialBandwidth, bandwidthConfig); - - // Picker-driven initial pick: when the slot is empty and the - // caller supplied a `picker`, defer to it instead of the - // bandwidth-aware default. The picker sees the full - // presentation (not narrowed by `userVideoTrackSelection`) — - // honoring the filter is the picker's responsibility when - // overridden. Returning `undefined` falls through to the - // bandwidth-aware default (graceful fallback). - // - // ABR re-evaluation (downgrade/upgrade by bandwidth) runs as - // usual on subsequent effect re-runs once the slot is set. - if (!selectedId && config.picker) { - const id = config.picker(presentation, config); - if (id) { - state[selectionKey].set(id); - return; - } - } - // `selectOptimal` decides the track to apply now given current - // selection + bandwidth + tuning. It returns: - // - the optimal when no current track or on a downgrade - // - the optimal when an upgrade clears `upgradeMargin` - // - the current track itself when an upgrade doesn't clear margin - // (caller's id-compare below no-ops in that case) - // Outer `?? selectLowestQuality` is defensive — `selectQuality` falls - // back to lowest internally; this catches future impls that don't. - // `selectLowestQuality` returns `undefined` only on empty input, - // and `candidates.length >= 2` here. - const currentTrack = candidates.find((t) => t.id === selectedId); - const optimal = - selectOptimal(candidates, bandwidth, { safetyMargin, upgradeMargin, currentTrack }) ?? - selectLowestQuality(candidates)!; - if (optimal.id !== selectedId) state[selectionKey].set(optimal.id); - }, - ], - }, - }, - }); -} - -// ============================================================================ -// Specialized exports — one per ABR-enabled track type -// ============================================================================ - -/** - * Manage `selectedVideoTrackId`: pick a default on src load, dynamically - * adjust based on bandwidth, clear on src unload. Honors - * `userVideoTrackSelection` as a partial-track constraint on candidates; - * short-circuits ABR when the constraint narrows to a single track. - * - * @example - * const reactor = switchVideoQuality.setup({ state }); - */ -export const switchVideoQuality = defineBehavior({ - stateKeys: ['presentation', 'bandwidthState', 'selectedVideoTrackId', 'userVideoTrackSelection'], - contextKeys: [], - setup: ({ - state, - config, - }: { - state: QualitySwitchingStateMap<'selectedVideoTrackId', 'userVideoTrackSelection'>; - config?: QualitySwitchingConfig; - }) => - setupQualitySwitching<'selectedVideoTrackId', 'userVideoTrackSelection', PartiallyResolvedVideoTrack | VideoTrack>({ - state, - config: { - ...config, - selectionKey: 'selectedVideoTrackId', - userSelectionKey: 'userVideoTrackSelection', - getTracks: (presentation) => - getTracksByType(presentation, 'video') as readonly (PartiallyResolvedVideoTrack | VideoTrack)[], - selectOptimal: selectQuality, - }, - }), -}); diff --git a/packages/spf/src/playback/behaviors/select-tracks.ts b/packages/spf/src/playback/behaviors/select-tracks.ts index 3d64fdaa..18b181e3 100644 --- a/packages/spf/src/playback/behaviors/select-tracks.ts +++ b/packages/spf/src/playback/behaviors/select-tracks.ts @@ -1,28 +1,33 @@ /** - * **Default audio/text track selection on src load / unselect on src unload.** - * When a presentation is resolved, sets `selectedAudioTrackId` / - * `selectedTextTrackId` to a per-type-picker default if no selection already - * exists. When the presentation is unset/reset (transitions back to - * unresolved), clears the selection so a stale id from the previous source - * doesn't persist. + * **Default audio/text/video track selection on src load / unselect on src unload.** + * When a presentation is resolved, sets `selectedVideoTrackId` / + * `selectedAudioTrackId` / `selectedTextTrackId` to a per-type-picker default + * if no selection already exists. When the presentation is unset/reset + * (transitions back to unresolved), clears the selection so a stale id from + * the previous source doesn't persist. * - * Lifecycle-driven: each transition fires its work once. Does not police - * the selection between transitions; external writes (user picks) are left - * alone. + * Lifecycle-driven: each transition fires its work once. Does not police the + * selection between transitions; external writes (user picks, ABR, programmatic + * filter-driven re-picks) are left alone. * * Picker is config-driven: each per-type export wires a sensible default - * (`pickFirstTrackId` for audio, `pickTextTrack` for text) and the caller - * can supply their own via `config.picker` for custom selection logic - * (language preferences, default-track handling, etc.). The behavior's + * (`pickAudioTrack` for audio — three-tier language-aware; `pickTextTrack` + * for text; `pickFirstTrackId` for video) and the caller can supply their + * own via `config.picker` for custom selection logic. The behavior's * `config` is forwarded to the picker as its second argument, so options * like `preferredAudioLanguage` / `preferredSubtitleLanguage` reach the * picker without an intermediate wrapping layer. * * Compose `selectVideoTrack` for the simple "pick a default video track" - * behavior, or `switchVideoQuality` (`./quality-switching.ts`) for the - * ABR-driven variant — they're alternatives, not stackable (both write - * `selectedVideoTrackId`). The simple variant tree-shakes out the - * ABR machinery (bandwidth-estimator, quality-selection algorithms). + * behavior, or `switchVideoTrack` (`./track-switching.ts`) for the + * ABR-driven variant. Compose `selectAudioTrack` for the simple default + * pick, or `switchAudioTrack` (`./track-switching.ts`) for the + * filter-reactive + mid-stream-flush slot-owner variant — when audio-abr + * lands, `switchAudioTrack` extends into `switchAudioQuality`. Compose + * only one per type — they're alternatives, not stackable (each writes + * the same `selected*TrackId` slot). The simple variants tree-shake out + * the heavier machinery (bandwidth estimator, quality selection, flush + * orchestration). */ import { defineBehavior } from '../../core/composition/create-composition'; @@ -30,6 +35,7 @@ import { createMachineReactor } from '../../core/reactors/create-machine-reactor import { computed, type ReadonlySignal, type Signal } from '../../core/signals/primitives'; import { type AudioSelectionConfig, + pickAudioTrack, pickFirstTrackId, pickTextTrack, type TextSelectionConfig, @@ -110,36 +116,18 @@ function setupTrackSelection({ } // ============================================================================ -// Per-helper-per-type configs — defaults that variants spread engine config over +// Default pickers // -// The engine-facing config carries `picker?` (optional override) plus -// fields the default picker consults (preferred language, etc.). The -// variant resolves the final picker (engine override or default) and -// forwards the whole engine config as `pickerConfig` so rich pickers can -// read their options. +// Each variant resolves its picker as `config?.picker ?? ` and +// forwards the whole engine config as `pickerConfig`, so rich pickers +// (`pickAudioTrack`, `pickTextTrack`) read their options directly. Audio +// and text use their primitive pickers as-is; video adapts +// `pickFirstTrackId` (positional `type` arg) into the `TrackPicker` shape. // ============================================================================ /** Default video picker: first track in the video selection set. */ const defaultVideoPicker: TrackPicker = (presentation) => pickFirstTrackId(presentation, 'video'); -const VIDEO_TRACK_SELECTION_CONFIG = { - ...VIDEO_TYPE_CONFIG, - picker: defaultVideoPicker, -} as const; - -/** Default audio picker: first track in the audio selection set. */ -const defaultAudioPicker: TrackPicker = (presentation) => pickFirstTrackId(presentation, 'audio'); - -const AUDIO_TRACK_SELECTION_CONFIG = { - ...AUDIO_TYPE_CONFIG, - picker: defaultAudioPicker, -} as const; - -const TEXT_TRACK_SELECTION_CONFIG = { - ...TEXT_TYPE_CONFIG, - picker: pickTextTrack, -} as const; - // ============================================================================ // Specialized exports — one per track type // ============================================================================ @@ -148,7 +136,7 @@ const TEXT_TRACK_SELECTION_CONFIG = { * Config for `selectVideoTrack`. Pass `picker` to fully override selection * logic; otherwise the default `pickFirstTrackId` is used. */ -export interface SelectVideoTrackConfig extends Omit { +export interface SelectVideoTrackConfig extends VideoSelectionConfig { picker?: TrackPicker; } @@ -156,7 +144,7 @@ export interface SelectVideoTrackConfig extends Omit { +export interface SelectAudioTrackConfig extends AudioSelectionConfig { picker?: TrackPicker; } @@ -194,6 +181,15 @@ export interface SelectAudioTrackConfig extends Omit { reactor.destroy(); }); - it.skip('uses preferred language configuration', async () => { + it('picks track matching preferredAudioLanguage when supplied', async () => { + const audioTracks: PartiallyResolvedAudioTrack[] = [ + { + type: 'audio', + id: 'audio-en', + url: 'http://example.com/audio-en.m3u8', + bandwidth: 128_000, + mimeType: 'audio/mp4', + codecs: ['mp4a.40.2'], + groupId: 'audio', + name: 'English', + language: 'en', + sampleRate: 48000, + channels: 2, + }, + { + type: 'audio', + id: 'audio-es', + url: 'http://example.com/audio-es.m3u8', + bandwidth: 128_000, + mimeType: 'audio/mp4', + codecs: ['mp4a.40.2'], + groupId: 'audio', + name: 'Spanish', + language: 'es', + sampleRate: 48000, + channels: 2, + }, + ]; + + const presentation = createPresentation({ audio: audioTracks }); + const state = makeState({ presentation }); + + const reactor = selectAudioTrack.setup({ state, config: { preferredAudioLanguage: 'es' } }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(state.selectedAudioTrackId.get()).toBe('audio-es'); + + reactor.destroy(); + }); + + it('falls back to DEFAULT=YES track when preferredAudioLanguage does not match', async () => { + const audioTracks: PartiallyResolvedAudioTrack[] = [ + { + type: 'audio', + id: 'audio-en', + url: 'http://example.com/audio-en.m3u8', + bandwidth: 128_000, + mimeType: 'audio/mp4', + codecs: ['mp4a.40.2'], + groupId: 'audio', + name: 'English', + language: 'en', + sampleRate: 48000, + channels: 2, + }, + { + type: 'audio', + id: 'audio-fr', + url: 'http://example.com/audio-fr.m3u8', + bandwidth: 128_000, + mimeType: 'audio/mp4', + codecs: ['mp4a.40.2'], + groupId: 'audio', + name: 'French', + language: 'fr', + default: true, + sampleRate: 48000, + channels: 2, + }, + ]; + + const presentation = createPresentation({ audio: audioTracks }); + const state = makeState({ presentation }); + + // Preferred language 'xx' has no match; picker falls back to the default + // track ('audio-fr') rather than the first track ('audio-en'). + const reactor = selectAudioTrack.setup({ state, config: { preferredAudioLanguage: 'xx' } }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(state.selectedAudioTrackId.get()).toBe('audio-fr'); + + reactor.destroy(); + }); + + it('falls back to first track when no language preference and no DEFAULT track', async () => { const audioTracks: PartiallyResolvedAudioTrack[] = [ { type: 'audio', @@ -261,7 +348,7 @@ describe('selectAudioTrack', () => { await new Promise((resolve) => setTimeout(resolve, 50)); - expect(state.selectedAudioTrackId.get()).toBe('audio-es'); + expect(state.selectedAudioTrackId.get()).toBe('audio-en'); reactor.destroy(); }); diff --git a/packages/spf/src/playback/behaviors/tests/quality-switching.test.ts b/packages/spf/src/playback/behaviors/tests/track-switching.test.ts similarity index 63% rename from packages/spf/src/playback/behaviors/tests/quality-switching.test.ts rename to packages/spf/src/playback/behaviors/tests/track-switching.test.ts index a4630b64..44a9e2ea 100644 --- a/packages/spf/src/playback/behaviors/tests/quality-switching.test.ts +++ b/packages/spf/src/playback/behaviors/tests/track-switching.test.ts @@ -1,7 +1,9 @@ -import { describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { StateSignals } from '../../../core/composition/create-composition'; import { signal } from '../../../core/signals/primitives'; import type { + AudioSelectionSet, + AudioTrack, MaybeResolvedPresentation, PartiallyResolvedVideoTrack, Presentation, @@ -9,13 +11,25 @@ import type { VideoTrack, } from '../../../media/types'; import type { BandwidthState } from '../../../network/bandwidth-estimator'; -import { type QualitySwitchingConfig, type QualitySwitchingState, switchVideoQuality } from '../quality-switching'; +import { + switchAudioTrack, + switchVideoTrack, + type TrackSwitchingConfig, + type TrackSwitchingState, +} from '../track-switching'; // ============================================================================ // Test helpers // ============================================================================ -function makeState(initial: QualitySwitchingState = {}): StateSignals { +interface SwitchVideoTrackState { + presentation?: MaybeResolvedPresentation; + bandwidthState?: BandwidthState; + selectedVideoTrackId?: string; + userVideoTrackSelection?: Partial; +} + +function makeState(initial: Partial = {}): StateSignals { return { presentation: signal(initial.presentation), bandwidthState: signal(initial.bandwidthState), @@ -70,15 +84,15 @@ const tracks = [ ]; // ============================================================================ -// switchVideoQuality +// switchVideoTrack // ============================================================================ -describe('switchVideoQuality', () => { +describe('switchVideoTrack', () => { describe('lifecycle (presentation-unresolved ↔ presentation-resolved)', () => { it('does nothing without a presentation', async () => { const state = makeState({ bandwidthState: createBandwidthState(3_000_000) }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBeUndefined(); reactor.destroy(); @@ -90,7 +104,7 @@ describe('switchVideoQuality', () => { bandwidthState: createBandwidthState(3_000_000), }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -107,7 +121,7 @@ describe('switchVideoQuality', () => { bandwidthState: createBandwidthState(3_000_000), }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -119,7 +133,7 @@ describe('switchVideoQuality', () => { it('re-picks default after src reset (presentation undefined → new resolved)', async () => { const state = makeState({ presentation: createPresentation(tracks) }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); // initialBandwidth-driven default — see "default-pick" describe. expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -141,7 +155,7 @@ describe('switchVideoQuality', () => { it('picks the initialBandwidth-optimal track when bandwidthState is undefined and no selection', async () => { const state = makeState({ presentation: createPresentation(tracks) }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); // Default initialBandwidth is 5 Mbps; 1080p (4.8 Mbps) requires // 5.65 Mbps with safetyMargin 0.85, so the optimal is 720p. @@ -158,7 +172,7 @@ describe('switchVideoQuality', () => { // unified with the pre-trust path. const state = makeState({ presentation: createPresentation(tracks), selectedVideoTrackId: '1080p' }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -174,7 +188,7 @@ describe('switchVideoQuality', () => { userVideoTrackSelection: { id: '360p' }, }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); // Bandwidth would normally select 1080p, but the filter locks to 360p. expect(state.selectedVideoTrackId.get()).toBe('360p'); @@ -189,7 +203,7 @@ describe('switchVideoQuality', () => { userVideoTrackSelection: { id: '720p' }, }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -216,7 +230,7 @@ describe('switchVideoQuality', () => { userVideoTrackSelection: { id: '360p' }, }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('360p'); @@ -235,7 +249,7 @@ describe('switchVideoQuality', () => { userVideoTrackSelection: { id: 'nonexistent-from-old-src' }, }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); // Filter matches zero tracks → fall back to full candidate set → // ABR picks optimal at 3 Mbps. @@ -257,7 +271,7 @@ describe('switchVideoQuality', () => { userVideoTrackSelection: { height: 1080 }, }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); // Among 1080a (3M) and 1080b (5.5M), 8 Mbps fits both; selectQuality // picks the highest bandwidth track (5.5M → 1080-high). @@ -275,7 +289,7 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: '720p', }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -293,7 +307,7 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: '360p', }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('1080p'); @@ -313,7 +327,7 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: '720p', }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -333,7 +347,7 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: 'low', }); - const reactor = switchVideoQuality.setup({ state }); + const reactor = switchVideoTrack.setup({ state }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('low'); @@ -349,8 +363,8 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: '360p', }); - const config: QualitySwitchingConfig = { quality: { safetyMargin: 1.0 } }; - const reactor = switchVideoQuality.setup({ state, config }); + const config: TrackSwitchingConfig = { quality: { safetyMargin: 1.0 } }; + const reactor = switchVideoTrack.setup({ state, config }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); reactor.destroy(); @@ -364,8 +378,8 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: 'low', }); - const config: QualitySwitchingConfig = { quality: { upgradeMargin: 1.05 } }; - const reactor = switchVideoQuality.setup({ state, config }); + const config: TrackSwitchingConfig = { quality: { upgradeMargin: 1.05 } }; + const reactor = switchVideoTrack.setup({ state, config }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('high'); @@ -387,8 +401,8 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: '360p', }); - const config: QualitySwitchingConfig = { initialBandwidth: 5_000_000 }; - const reactor = switchVideoQuality.setup({ state, config }); + const config: TrackSwitchingConfig = { initialBandwidth: 5_000_000 }; + const reactor = switchVideoTrack.setup({ state, config }); await flush(); expect(state.selectedVideoTrackId.get()).toBe('720p'); reactor.destroy(); @@ -403,8 +417,8 @@ describe('switchVideoQuality', () => { bandwidthState: createBandwidthState(6_000_000), }); - const config: QualitySwitchingConfig = { picker: () => '360p' }; - const reactor = switchVideoQuality.setup({ state, config }); + const config: TrackSwitchingConfig = { picker: () => '360p' }; + const reactor = switchVideoTrack.setup({ state, config }); await flush(); // Picker drives the initial selection. expect(state.selectedVideoTrackId.get()).toBe('360p'); @@ -424,8 +438,8 @@ describe('switchVideoQuality', () => { bandwidthState: createBandwidthState(3_000_000), }); - const config: QualitySwitchingConfig = { picker: () => undefined }; - const reactor = switchVideoQuality.setup({ state, config }); + const config: TrackSwitchingConfig = { picker: () => undefined }; + const reactor = switchVideoTrack.setup({ state, config }); await flush(); // 3 Mbps with default safetyMargin 0.85 → 720p (1080p needs 5.65 Mbps). expect(state.selectedVideoTrackId.get()).toBe('720p'); @@ -451,11 +465,11 @@ describe('switchVideoQuality', () => { selectedVideoTrackId: '720p', }); - const config: QualitySwitchingConfig = { + const config: TrackSwitchingConfig = { bandwidth: { minTotalBytes: 40_000 }, initialBandwidth: 5_000_000, }; - const reactor = switchVideoQuality.setup({ state, config }); + const reactor = switchVideoTrack.setup({ state, config }); await flush(); // With the override the 800 kbps measurement is trusted and drives a // downgrade to 360p; at the default threshold the 5 Mbps initial @@ -465,3 +479,193 @@ describe('switchVideoQuality', () => { }); }); }); + +// ============================================================================ +// switchAudioTrack +// +// The audio variant shares `setupTrackSwitching` with `switchVideoTrack` — +// these tests cover the audio-specific surface: default picker, language +// pinning, filter reactivity, single-candidate short-circuit. The +// bandwidth-driven re-evaluation tests live above under `switchVideoTrack` +// and don't need duplicating; audio's `selectAudioCurrent` pins to the +// current track and is exercised here by the steady-state assertions. +// ============================================================================ + +interface SwitchAudioTrackState { + presentation?: MaybeResolvedPresentation; + bandwidthState?: BandwidthState; + selectedAudioTrackId?: string; + userAudioTrackSelection?: Partial; +} + +function makeAudioState(initial: SwitchAudioTrackState = {}): StateSignals { + return { + presentation: signal(initial.presentation), + bandwidthState: signal(initial.bandwidthState), + selectedAudioTrackId: signal(initial.selectedAudioTrackId), + userAudioTrackSelection: signal | undefined>(initial.userAudioTrackSelection), + }; +} + +function createAudioPresentation(tracks: AudioTrack[]): Presentation { + return { + id: 'pres-1', + url: 'http://example.com/playlist.m3u8', + selectionSets: [ + { + id: 'audio-set', + type: 'audio' as const, + switchingSets: [ + { + id: 'audio-switching', + type: 'audio' as const, + tracks, + }, + ], + } as AudioSelectionSet, + ], + startTime: 0, + }; +} + +function makeAudioTrack(id: string, overrides: Partial = {}): AudioTrack { + return { + type: 'audio', + id, + url: `http://example.com/${id}.m3u8`, + bandwidth: 128_000, + mimeType: 'audio/mp4', + codecs: ['mp4a.40.2'], + groupId: 'audio', + name: id, + sampleRate: 48000, + channels: 2, + startTime: 0, + duration: 10, + initialization: { url: `http://example.com/${id}-init.mp4` }, + segments: [], + ...overrides, + }; +} + +describe('switchAudioTrack', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('selects the first audio track when no preference or filter', async () => { + const state = makeAudioState({ + presentation: createAudioPresentation([makeAudioTrack('audio-en', { language: 'en' })]), + }); + + const reactor = switchAudioTrack.setup({ state }); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(state.selectedAudioTrackId.get()).toBe('audio-en'); + + reactor.destroy(); + }); + + it('picks track matching preferredAudioLanguage when supplied', async () => { + const state = makeAudioState({ + presentation: createAudioPresentation([ + makeAudioTrack('audio-en', { language: 'en' }), + makeAudioTrack('audio-es', { language: 'es' }), + ]), + }); + + const reactor = switchAudioTrack.setup({ state, config: { preferredAudioLanguage: 'es' } }); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(state.selectedAudioTrackId.get()).toBe('audio-es'); + + reactor.destroy(); + }); + + it('clears selectedAudioTrackId on src unload', async () => { + const state = makeAudioState({ + presentation: createAudioPresentation([makeAudioTrack('audio-en', { language: 'en' })]), + }); + + const reactor = switchAudioTrack.setup({ state }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(state.selectedAudioTrackId.get()).toBe('audio-en'); + + state.presentation.set(undefined); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(state.selectedAudioTrackId.get()).toBeUndefined(); + + reactor.destroy(); + }); +}); + +describe('switchAudioTrack — userAudioTrackSelection filter', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('narrows candidates by filter (language)', async () => { + const state = makeAudioState({ + presentation: createAudioPresentation([ + makeAudioTrack('audio-en', { language: 'en' }), + makeAudioTrack('audio-es', { language: 'es' }), + ]), + userAudioTrackSelection: { language: 'es' }, + }); + + const reactor = switchAudioTrack.setup({ state }); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(state.selectedAudioTrackId.get()).toBe('audio-es'); + + reactor.destroy(); + }); + + it('re-picks on filter change mid-presentation', async () => { + const state = makeAudioState({ + presentation: createAudioPresentation([ + makeAudioTrack('audio-en', { language: 'en' }), + makeAudioTrack('audio-es', { language: 'es' }), + ]), + }); + + const reactor = switchAudioTrack.setup({ state }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(state.selectedAudioTrackId.get()).toBe('audio-en'); + + state.userAudioTrackSelection.set({ language: 'es' }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(state.selectedAudioTrackId.get()).toBe('audio-es'); + + reactor.destroy(); + }); + + it('filter narrowing to a single track short-circuits the picker', async () => { + const state = makeAudioState({ + presentation: createAudioPresentation([ + makeAudioTrack('audio-en', { language: 'en' }), + makeAudioTrack('audio-es', { language: 'es' }), + ]), + userAudioTrackSelection: { id: 'audio-es' }, + }); + + const reactor = switchAudioTrack.setup({ state }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(state.selectedAudioTrackId.get()).toBe('audio-es'); + + reactor.destroy(); + }); + + it('empty filter result falls back to unfiltered candidate set', async () => { + const state = makeAudioState({ + presentation: createAudioPresentation([makeAudioTrack('audio-en', { language: 'en' })]), + userAudioTrackSelection: { language: 'es' }, // no Spanish track exists + }); + + const reactor = switchAudioTrack.setup({ state }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(state.selectedAudioTrackId.get()).toBe('audio-en'); + + reactor.destroy(); + }); +}); diff --git a/packages/spf/src/playback/behaviors/track-switching.ts b/packages/spf/src/playback/behaviors/track-switching.ts new file mode 100644 index 00000000..6466e4f2 --- /dev/null +++ b/packages/spf/src/playback/behaviors/track-switching.ts @@ -0,0 +1,438 @@ +/** + * **Per-type track-selection slot management with optional ABR.** While a + * presentation is resolved, owns the selection slot's lifecycle: pick a + * default, react to user intent (partial-track filter), re-evaluate + * algorithmically (today: bandwidth via `selectQuality` for video; pin-to- + * current for audio), and clear on src unload. + * + * User intent is expressed as a partial-track description in a sibling slot + * (e.g. `userVideoTrackSelection`, `userAudioTrackSelection`) which + * constrains the candidate set for selection — when the constraint narrows + * candidates to exactly one, the choice is fully determined and the + * selection algorithm is short-circuited (no bandwidth read, no effect + * re-fire on bandwidth changes). + * + * Lifecycle: `'presentation-unresolved'` ↔ `'presentation-resolved'`. The + * `'presentation-resolved'` state owns the selection slot; its + * entry-returned cleanup clears the slot on exit (canonical + * cleanup-binds-to-setup per `reactors.md`). + * + * Hysteresis (video ABR today, audio ABR when added): downgrades apply + * immediately; upgrades require the optimal track's bandwidth to exceed + * the current track's by `upgradeMargin`. No temporal state — short-term + * smoothing is the bandwidth estimator's job. + * + * Initial pick is configurable via `config.picker` — pass any `TrackPicker` + * to override the default-pick for the empty-slot case. Algorithmic + * re-evaluation (`selectOptimal`) is unaffected. + * + * Variants: `switchVideoTrack` (bandwidth-driven `selectOptimal`), + * `switchAudioTrack` (pin-to-current `selectOptimal`, ABR-ready shape for + * a future bandwidth-driven variant once audio-ABR lands). Both variants + * share this helper; the only point of variation is the per-variant + * `selectOptimal` and the candidate-track type. Future track-switching + * axes (text tracks, etc.) plug in the same way. + */ + +import { defineBehavior } from '../../core/composition/create-composition'; +import { createMachineReactor } from '../../core/reactors/create-machine-reactor'; +import { computed, peek, type ReadonlySignal, type Signal } from '../../core/signals/primitives'; +import { + DEFAULT_QUALITY_CONFIG, + type QualityConfig, + selectLowestQuality, + selectQuality, +} from '../../media/abr/quality-selection'; +import { pickAudioTrack, type TrackPicker } from '../../media/primitives/select-tracks'; +import { + type AudioTrack, + isResolvedPresentation, + type MaybeResolvedPresentation, + type PartiallyResolvedAudioTrack, + type PartiallyResolvedVideoTrack, + type VideoTrack, +} from '../../media/types'; +import { getTracksByType } from '../../media/utils/tracks'; +import type { BandwidthConfig, BandwidthState } from '../../network/bandwidth-estimator'; +import { DEFAULT_BANDWIDTH_CONFIG, getBandwidthEstimate } from '../../network/bandwidth-estimator'; + +// ============================================================================ +// State + Config +// ============================================================================ + +export interface TrackSwitchingState { + presentation?: MaybeResolvedPresentation; + bandwidthState?: BandwidthState; + selectedVideoTrackId?: string; + selectedAudioTrackId?: string; + /** + * Partial-track description expressing user intent for video. When set, + * narrows candidates to tracks matching every present field. Common case + * is `{ id: 'specific-track-id' }` for "manual quality"; other shapes + * work (e.g., `{ height: 720 }` constrains to 720p tracks — ABR + * continues to pick among them). + * + * When narrowed candidates contain exactly one track, ABR is + * short-circuited entirely (no bandwidth read, no effect re-fire). + * + * Falls back to the unfiltered set when the filter matches no tracks + * (e.g., user-picked id from a previous source doesn't exist here). + */ + userVideoTrackSelection?: Partial; + /** + * Partial-track description expressing user intent for audio. Common + * case is `{ language: 'es' }` for language-pinning, `{ id: 'X' }` for + * absolute pinning. Same narrowing + short-circuit + fallback semantics + * as `userVideoTrackSelection`. + */ + userAudioTrackSelection?: Partial; +} + +export interface TrackSwitchingConfig { + /** + * Quality-selection tuning consumed by bandwidth-driven `selectOptimal` + * variants (today: `switchVideoTrack`'s `selectQuality`). `safetyMargin` + * is the bandwidth-headroom multiplier; `upgradeMargin` is the + * hysteresis ratio gating upgrades. Defaults: `DEFAULT_QUALITY_CONFIG` + * (0.85 / 1.15). Ignored by pin-to-current variants + * (today: `switchAudioTrack`). + */ + quality?: Partial; + + /** + * Bandwidth-estimator tuning passed through to `getBandwidthEstimate`. + * Merged over `DEFAULT_BANDWIDTH_CONFIG`. Consumed only by bandwidth- + * driven variants. + */ + bandwidth?: Partial; + + /** + * Bandwidth estimate in bps to use before enough samples have been + * collected. Default: 5_000_000 (5 Mbps). + */ + initialBandwidth?: number; + + /** + * Override the initial-pick algorithm. When set, the picker is called + * the first time the slot is empty in the `'presentation-resolved'` + * state; its returned id is set verbatim (no algorithmic logic). + * Subsequent re-evaluation via the variant's `selectOptimal` is + * unaffected. + * + * Honors of the user-selection filter are the picker's responsibility + * when overridden. If the picker returns `undefined`, the variant's + * default initial pick fires (graceful fallback). + */ + picker?: TrackPicker; + + /** + * Audio-variant config — preferred language consumed by the default + * audio picker (`pickAudioTrack`). Ignored by other variants. + */ + preferredAudioLanguage?: string; +} + +/** Default initial-bandwidth value before bandwidth measurements arrive. */ +export const DEFAULT_INITIAL_BANDWIDTH = 5_000_000; + +// ============================================================================ +// Specialization helper +// +// `setupTrackSwitching` has the same shape as a Behavior `setup` function: +// `({ state, config }) => Reactor`. Each `switchXTrack` export below calls +// it from inside its own `defineBehavior` setup, passing the per-type slot +// keys, track type, and selection algorithm via three generic parameters — +// `S` (selection slot key), `U` (user-selection slot key), `T` (candidate +// track type). +// +// -- Design note: why narrow `SelectionKey` / `UserSelectionKey` unions ---- +// Goal we did not reach: have callers "fully pass in" the slot keys, with +// the helper enforcing zero internal knowledge of which literals are valid. +// What blocks it: indexing a mapped-type intersection by a generic key. +// When `S extends keyof TrackSwitchingState` (or `string`), TS conservatively +// treats `state[selectionKey]` as the union of every possible match across +// the intersected mapped portions — including the fixed-key signals +// (`presentation`, `bandwidthState`) — and widens to their value-type union. +// The sibling pattern hits the same constraint and answers it the same way: +// `SelectedTrackKey` in `select-tracks.ts` is a hardcoded narrow union for +// the same reason. +// +// Current pick is the narrow-union route because it matches siblings and +// the unions read as documentation ("these are the slots this helper +// manages") rather than restriction. Extending to a new track-switching +// axis is one literal per union. +// -------------------------------------------------------------------------- +// ============================================================================ + +/** Minimum candidate-track shape consumed by the helper. */ +type SwitchableTrack = { id: string; bandwidth?: number }; + +type SelectionKey = 'selectedVideoTrackId' | 'selectedAudioTrackId'; +type UserSelectionKey = 'userVideoTrackSelection' | 'userAudioTrackSelection'; + +// Each mapped value references `P` so TS keeps the per-key dependency and +// resolves `state[selectionKey]` / `state[userSelectionKey]` to the right +// arm. `T` (track type) deliberately stays out of the state map — pulling +// it in detaches the user-selection mapped value from `P` and TS collapses +// the intersection. T flows through `TrackSwitchingSetupConfig` instead; +// the user-filter access casts at the read site (see below). +type TrackSwitchingStateMap = { + presentation: ReadonlySignal; + bandwidthState: ReadonlySignal; +} & { [P in S]: Signal } & { [P in U]: ReadonlySignal }; + +/** + * Selection context passed to `selectOptimal`. Built once per effect run. + * Bandwidth-aware variants (video ABR, future audio ABR) read all fields; + * pin-to-current variants (audio today) ignore the bandwidth-shaped fields. + * + * The context is built *inside* the effect, so bandwidth-aware variants + * subscribe to `bandwidthState` automatically; pin-to-current variants + * receive the same context but never re-fire on bandwidth changes because + * the single-candidate short-circuit (above) bypasses the bandwidth read. + */ +export interface SelectionCtx { + bandwidth: number; + safetyMargin: number; + upgradeMargin: number; + currentTrack?: T; +} + +interface TrackSwitchingSetupConfig + extends TrackSwitchingConfig { + selectionKey: S; + userSelectionKey: U; + getTracks: (presentation: MaybeResolvedPresentation) => readonly T[]; + selectOptimal: (tracks: readonly T[], ctx: SelectionCtx) => T | undefined; +} + +function setupTrackSwitching({ + state, + config, +}: { + state: TrackSwitchingStateMap; + config: TrackSwitchingSetupConfig; +}) { + const safetyMargin = config.quality?.safetyMargin ?? DEFAULT_QUALITY_CONFIG.safetyMargin; + const upgradeMargin = config.quality?.upgradeMargin ?? DEFAULT_QUALITY_CONFIG.upgradeMargin; + const initialBandwidth = config.initialBandwidth ?? DEFAULT_INITIAL_BANDWIDTH; + const bandwidthConfig: BandwidthConfig = { ...DEFAULT_BANDWIDTH_CONFIG, ...config.bandwidth }; + const { selectionKey, userSelectionKey, getTracks, selectOptimal } = config; + + const derivedStateSignal = computed(() => + isResolvedPresentation(state.presentation.get()) + ? ('presentation-resolved' as const) + : ('presentation-unresolved' as const) + ); + + return createMachineReactor({ + initial: 'presentation-unresolved', + monitor: () => derivedStateSignal.get(), + states: { + 'presentation-unresolved': {}, + 'presentation-resolved': { + // Canonical cleanup-binds-to-setup: the selection slot's valid + // lifespan is exactly 'presentation-resolved'. Clear fires on + // 'presentation-resolved' exit, covering both src unload and + // behavior destroy. + entry: () => () => state[selectionKey].set(undefined), + effects: [ + () => { + const presentation = peek(state.presentation); + if (!presentation) return; + + const allTracks = getTracks(presentation); + const [firstAllTrack] = allTracks; + if (!firstAllTrack) return; + + // State stores the filter as `Partial` (user-facing + // shape — includes per-type fields like `height` or + // `language`); the helper works against `Partial` so + // filter and track access share one index type. Filter keys + // absent on a partially-resolved track read as `undefined` + // and just exclude that track. + const userFilter = state[userSelectionKey].get() as Partial | undefined; + const matching = userFilter + ? allTracks.filter((track) => { + for (const key in userFilter) { + const filterValue = userFilter[key as keyof T]; + if (filterValue !== undefined && track[key as keyof T] !== filterValue) return false; + } + return true; + }) + : allTracks; + // Fall back to all tracks when the filter excludes everything + // (e.g., user-picked id doesn't exist in the current source). + const candidates = matching.length > 0 ? matching : allTracks; + if (!candidates.length) return; + + const selectedId = state[selectionKey].get(); + + // Common case: user has fully constrained the choice (e.g., + // `{ id: 'specific-720p' }` or `{ language: 'es' }` when only + // one Spanish track exists). Skip the algorithm path — and + // crucially, don't read `bandwidthState` so the effect + // doesn't re-fire on bandwidth changes while the user's + // selection holds. + if (candidates.length === 1) { + if (candidates[0]!.id !== selectedId) state[selectionKey].set(candidates[0]!.id); + return; + } + + // Read bandwidth up front to establish the signal subscription — + // future bandwidth changes must re-fire this effect even when + // the picker branch below takes the early-return path. (If the + // picker bails out without ever touching `bandwidthState`, + // bandwidth-driven `selectOptimal` would otherwise be deaf to + // subsequent bandwidth changes.) Pin-to-current variants read + // the field but ignore it — the subscription cost is fixed + // per effect run regardless. + // + // Single path for pre-trust and post-trust: `getBandwidthEstimate` + // returns `initialBandwidth` when state is undefined or bytes + // sampled hasn't crossed `minTotalBytes`, so the initial pick + // and early-ABR window run the same `selectOptimal` path as a + // fully-trusted measurement. + const bandwidth = getBandwidthEstimate(state.bandwidthState.get(), initialBandwidth, bandwidthConfig); + + // Picker-driven initial pick: when the slot is empty and the + // caller supplied a `picker`, defer to it instead of the + // algorithmic default. The picker sees the full presentation + // (not narrowed by the user-selection filter) — honoring the + // filter is the picker's responsibility when overridden. + // Returning `undefined` falls through to the algorithmic + // default pick (graceful fallback). + // + // Algorithmic re-evaluation runs as usual on subsequent + // effect re-runs once the slot is set. + if (!selectedId && config.picker) { + const id = config.picker(presentation, config); + if (id) { + state[selectionKey].set(id); + return; + } + } + // `selectOptimal` decides the track to apply now given current + // selection + context. Returns: + // - the optimal when no current track or on a downgrade + // - the optimal when an upgrade clears `upgradeMargin` + // - the current track itself when an upgrade doesn't clear + // margin (caller's id-compare below no-ops in that case) + // Outer `?? selectLowestQuality` is defensive — `selectQuality` + // falls back to lowest internally; pin-to-current variants + // return `currentTrack ?? tracks[0]` so they never produce + // `undefined` for a non-empty `candidates`. The fallback + // catches future variants that don't return a definitive pick. + const currentTrack = candidates.find((t) => t.id === selectedId); + const ctx: SelectionCtx = { bandwidth, safetyMargin, upgradeMargin, currentTrack }; + const optimal = selectOptimal(candidates, ctx) ?? selectLowestQualityWithBandwidth(candidates); + if (optimal && optimal.id !== selectedId) state[selectionKey].set(optimal.id); + }, + ], + }, + }, + }); +} + +/** + * Adapter for `selectLowestQuality` that tolerates tracks whose `bandwidth` + * field is optional (audio's candidate shape). Falls back to the first + * candidate when no bandwidth info is available. + */ +function selectLowestQualityWithBandwidth(tracks: readonly T[]): T | undefined { + if (tracks.length === 0) return undefined; + const withBandwidth = tracks.filter((t): t is T & { bandwidth: number } => typeof t.bandwidth === 'number'); + if (withBandwidth.length === 0) return tracks[0]; + return selectLowestQuality(withBandwidth); +} + +// ============================================================================ +// Variant: switchVideoTrack — bandwidth-driven ABR +// ============================================================================ + +type VideoTrackCandidate = PartiallyResolvedVideoTrack | VideoTrack; + +/** + * Manage `selectedVideoTrackId`: pick a default on src load, dynamically + * adjust based on bandwidth, clear on src unload. Honors + * `userVideoTrackSelection` as a partial-track constraint on candidates; + * short-circuits ABR when the constraint narrows to a single track. + * + * @example + * const reactor = switchVideoTrack.setup({ state }); + */ +export const switchVideoTrack = defineBehavior({ + stateKeys: ['presentation', 'bandwidthState', 'selectedVideoTrackId', 'userVideoTrackSelection'], + contextKeys: [], + setup: ({ + state, + config, + }: { + state: TrackSwitchingStateMap<'selectedVideoTrackId', 'userVideoTrackSelection'>; + config?: TrackSwitchingConfig; + }) => + setupTrackSwitching<'selectedVideoTrackId', 'userVideoTrackSelection', VideoTrackCandidate>({ + state, + config: { + ...config, + selectionKey: 'selectedVideoTrackId', + userSelectionKey: 'userVideoTrackSelection', + getTracks: (presentation) => getTracksByType(presentation, 'video') as readonly VideoTrackCandidate[], + selectOptimal: selectQuality, + }, + }), +}); + +// ============================================================================ +// Variant: switchAudioTrack — pin-to-current (ABR-ready shape) +// ============================================================================ + +type AudioTrackCandidate = PartiallyResolvedAudioTrack | AudioTrack; + +/** + * Audio's `selectOptimal` — pin-to-current variant. Returns the current + * track if it's in the candidate set; otherwise the first candidate. No + * bandwidth-driven re-evaluation today (audio is not ABR-driven yet); the + * `ctx` shape carries bandwidth so audio-ABR can swap this for a + * bandwidth-aware variant without touching the helper. + */ +const selectAudioCurrent = ( + tracks: readonly AudioTrackCandidate[], + { currentTrack }: SelectionCtx +): AudioTrackCandidate | undefined => currentTrack ?? tracks[0]; + +/** + * Manage `selectedAudioTrackId`: pick a default on src load, narrow by + * `userAudioTrackSelection` filter, re-pick on filter change, clear on + * src unload. + * + * Mid-stream flush on language switch is handled by the segment-loader's + * `planTasks` (see `playback/actors/dom/segment-loader.ts`) — not this + * behavior. Same split as the video pipeline: slot owner writes; loader + * orchestrates segment + flush plans. + * + * @example + * const reactor = switchAudioTrack.setup({ state }); + */ +export const switchAudioTrack = defineBehavior({ + stateKeys: ['presentation', 'bandwidthState', 'selectedAudioTrackId', 'userAudioTrackSelection'], + contextKeys: [], + setup: ({ + state, + config, + }: { + state: TrackSwitchingStateMap<'selectedAudioTrackId', 'userAudioTrackSelection'>; + config?: TrackSwitchingConfig; + }) => + setupTrackSwitching<'selectedAudioTrackId', 'userAudioTrackSelection', AudioTrackCandidate>({ + state, + config: { + ...config, + selectionKey: 'selectedAudioTrackId', + userSelectionKey: 'userAudioTrackSelection', + getTracks: (presentation) => getTracksByType(presentation, 'audio') as readonly AudioTrackCandidate[], + selectOptimal: selectAudioCurrent, + picker: config?.picker ?? pickAudioTrack, + }, + }), +}); diff --git a/packages/spf/src/playback/engines/hls/engine-audio-only.ts b/packages/spf/src/playback/engines/hls/engine-audio-only.ts index 3dda7af9..05231dc9 100644 --- a/packages/spf/src/playback/engines/hls/engine-audio-only.ts +++ b/packages/spf/src/playback/engines/hls/engine-audio-only.ts @@ -8,7 +8,7 @@ import { makeShareSignals, type ShareSignalsConfig } from '../../../core/composi import type { BackBufferConfig } from '../../../media/buffer/back-buffer'; import type { ForwardBufferConfig } from '../../../media/buffer/forward-buffer'; import { parseMultivariantPlaylist } from '../../../media/hls/parse-multivariant'; -import type { MaybeResolvedPresentation } from '../../../media/types'; +import type { AudioTrack, MaybeResolvedPresentation } from '../../../media/types'; import { getResolvedSelectedTrackDuration } from '../../../media/utils/track-selection'; import type { SegmentLoaderActor } from '../../actors/dom/segment-loader'; import type { SourceBufferActor } from '../../actors/dom/source-buffer'; @@ -25,8 +25,8 @@ import { trackLoadTriggers } from '../../behaviors/dom/track-load-triggers'; import { updateMediaSourceDuration } from '../../behaviors/dom/update-mediasource-duration'; import { type ParsePresentation, resolvePresentation } from '../../behaviors/resolve-presentation'; import { resolveAudioTrack } from '../../behaviors/resolve-track'; -import { selectAudioTrack } from '../../behaviors/select-tracks'; import { syncPreload } from '../../behaviors/sync-preload'; +import { switchAudioTrack } from '../../behaviors/track-switching'; // ============================================================================ // Audio-Only HLS Engine State & Context @@ -43,6 +43,14 @@ export interface SimpleHlsAudioOnlyEngineState { presentation?: MaybeResolvedPresentation; preload?: 'auto' | 'metadata' | 'none'; selectedAudioTrackId?: string; + /** + * Consumer-driven constraint narrowing the audio candidate set. Sibling + * of `userVideoTrackSelection` in the default engine. Partial-track + * shape — `{ language: 'es' }`, `{ id: 'audio-en' }`, etc. + * `selectAudioTrack` reads this and re-picks when it changes. + * Multi-language-audio Tier 2 programmatic-write path. + */ + userAudioTrackSelection?: Partial; currentTime?: number; loadActivated?: boolean; } @@ -90,7 +98,7 @@ const shareSignals = makeShareSignals; + /** + * Consumer-driven constraint narrowing the audio candidate set. Sibling + * of `userVideoTrackSelection`. Partial-track shape — `{ language: 'es' }`, + * `{ id: 'audio-en' }`, etc. `selectAudioTrack` reads this and re-picks + * when it changes. Multi-language-audio Tier 2 programmatic-write path. + */ + userAudioTrackSelection?: Partial; currentTime?: number; loadActivated?: boolean; } @@ -237,9 +244,9 @@ export function createSimpleHlsEngine( resolvePresentation, // Track selection (reads config for initial preferences). - // Video selection lives in switchVideoQuality, which owns the - // default-pick + ABR-driven adjustment for selectedVideoTrackId. - selectAudioTrack, + // Video selection lives in switchVideoTrack (composed below); + // audio selection lives in switchAudioTrack (composed below) — + // both are slot owners with filter-reactivity, mirroring shapes. selectTextTrack, // Resolve selected tracks (fetch media playlists) @@ -262,7 +269,11 @@ export function createSimpleHlsEngine( // Playback tracking trackCurrentTime, - switchVideoQuality, + switchVideoTrack, + switchAudioTrack, + // Mid-stream audio-buffer flush on language switch is handled in + // `segment-loader`'s `planTasks` (predicate: language differs from + // the previously-buffered track) — not in switchAudioTrack itself. // Segment loading loadVideoSegments, @@ -283,7 +294,7 @@ export function createSimpleHlsEngine( ], { config: finalConfig, - // Seed bandwidthState so switchVideoQuality fires on initial subscribe + // Seed bandwidthState so switchVideoTrack fires on initial subscribe // with the `initialBandwidth` fallback rather than waiting for the // first chunk. The empty sample buffer means `getBandwidthEstimate` // returns the configured initial bandwidth until real samples land. diff --git a/packages/spf/src/playback/engines/hls/tests/engine-audio-only.test.ts b/packages/spf/src/playback/engines/hls/tests/engine-audio-only.test.ts index c7b0609a..b2433981 100644 --- a/packages/spf/src/playback/engines/hls/tests/engine-audio-only.test.ts +++ b/packages/spf/src/playback/engines/hls/tests/engine-audio-only.test.ts @@ -47,6 +47,21 @@ describe('createHlsAudioOnlyEngine', () => { engine.destroy(); }); + it('exposes userAudioTrackSelection slot for multi-language-audio Tier 2 writes', () => { + const engine = createHlsAudioOnlyEngine(); + + // Slot exists as a signal — consumer-facing programmatic-write path + // for multi-language-audio. + expect(engine.state.userAudioTrackSelection).toBeDefined(); + expect(typeof engine.state.userAudioTrackSelection.get).toBe('function'); + expect(typeof engine.state.userAudioTrackSelection.set).toBe('function'); + + engine.state.userAudioTrackSelection.set({ language: 'es' }); + expect(engine.state.userAudioTrackSelection.get()).toEqual({ language: 'es' }); + + engine.destroy(); + }); + it('does not seed bandwidthState (no ABR behavior subscribed at init)', () => { const engine = createHlsAudioOnlyEngine();