diff --git a/packages/spf/src/playback/actors/dom/segment-loader.ts b/packages/spf/src/playback/actors/dom/segment-loader.ts index a69ffe1f..a6c22b6c 100644 --- a/packages/spf/src/playback/actors/dom/segment-loader.ts +++ b/packages/spf/src/playback/actors/dom/segment-loader.ts @@ -13,6 +13,7 @@ import { type ForwardBufferConfig, getSegmentsToLoad, } from '../../../media/buffer/forward-buffer'; +import { readFirstBaseMediaDecodeTime, readFirstMediaTimescale } from '../../../media/mp4/timestamp-origin'; import { type AddressableObject, type AudioTrack, @@ -20,7 +21,13 @@ import { type Segment, type VideoTrack, } from '../../../media/types'; -import type { AppendInitMessage, AppendSegmentMessage, RemoveMessage, SourceBufferActor } from './source-buffer'; +import type { + AppendData, + AppendInitMessage, + AppendSegmentMessage, + RemoveMessage, + SourceBufferActor, +} from './source-buffer'; // ============================================================================ // BUFFER STATE TYPES @@ -102,6 +109,14 @@ export interface SegmentLoaderActorContext { inFlightInitTrackId: string | null; /** Segment ID currently being fetched/appended, or null. */ inFlightSegmentId: string | null; + /** + * Relocation offset in seconds (`−baseMediaDecodeTime / timescale`), or null + * until established / when relocation is off (spike). Exposed on the snapshot + * so the text path can consume the same offset to rebase cues — Mux cues carry + * no `X-TIMESTAMP-MAP`, so text can't self-derive the origin and reads the + * A/V-established one instead. + */ + relocationOffset: number | null; } export type SegmentLoaderActor = MessageActor; @@ -114,6 +129,23 @@ export type SegmentLoaderActor = MessageActor; backBuffer?: Partial; + /** + * Non-zero-PTS relocation (spike). When true, this loader reads its track's + * decode-time origin (`tfdt`/`mdhd`) from the init + first media segment and + * relocates the buffer to 0-based via `timestampOffset`. Per-track single-origin + * (Tier-1); off by default. + */ + relocateTimestampOrigin?: boolean; +} + +/** Per-track relocation working state (spike). Established once, on first media segment. */ +interface RelocationState { + readonly enabled: boolean; + /** `mdhd` timescale from the init segment. */ + timescale?: number; + /** `−(baseMediaDecodeTime / timescale)` — the SourceBuffer timestampOffset. */ + offset?: number; + established: boolean; } // ============================================================================ @@ -168,6 +200,28 @@ function waitForIdle(snapshot: SourceBufferActor['snapshot'], signal: AbortSigna }); } +/** Drain an async byte-iterable into one contiguous buffer (relocation spike). */ +async function collect(iterable: AsyncIterable): Promise { + const chunks: Uint8Array[] = []; + let total = 0; + for await (const chunk of iterable) { + chunks.push(chunk); + total += chunk.length; + } + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + +/** A collected buffer spans its backing ArrayBuffer exactly — hand it to MSE as one append. */ +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + return bytes.buffer as ArrayBuffer; +} + // ============================================================================ // LOAD TASK FACTORY // ============================================================================ @@ -177,6 +231,7 @@ interface LoadTaskOptions { setContext: (ctx: SegmentLoaderActorContext) => void; fetchBytes: FetchBytes; sourceBufferActor: SourceBufferActor; + relocation: RelocationState; } /** @@ -187,7 +242,7 @@ interface LoadTaskOptions { */ function makeLoadTask( op: LoadTask, - { getContext, setContext, fetchBytes, sourceBufferActor }: LoadTaskOptions + { getContext, setContext, fetchBytes, sourceBufferActor, relocation }: LoadTaskOptions ): Task { return new Task(async (taskSignal) => { if (taskSignal.aborted) return; @@ -203,7 +258,15 @@ function makeLoadTask( try { // Init segments are small and need the full body before appending. // minChunkSize: Infinity accumulates all chunks into one before yielding. - const data = await fetchBytes(op, { signal: taskSignal, minChunkSize: Infinity }); + const body = await fetchBytes(op, { signal: taskSignal, minChunkSize: Infinity }); + // Relocation (spike): read this track's mdhd timescale from the init — + // half of the decode-time origin (paired with the first segment's tfdt). + let data: AppendData = body; + if (relocation.enabled) { + const bytes = await collect(body); + relocation.timescale = readFirstMediaTimescale(bytes); + data = toArrayBuffer(bytes); + } if (!taskSignal.aborted) { sourceBufferActor.send({ type: 'append-init', data, meta: op.meta }); await waitForIdle(sourceBufferActor.snapshot, taskSignal); @@ -219,10 +282,32 @@ function makeLoadTask( // directly to the actor so chunks are appended as they arrive. setContext({ ...getContext(), inFlightSegmentId: op.meta.id }); try { - const stream = await fetchBytes(op, { signal: taskSignal }); - if (!taskSignal.aborted) { - sourceBufferActor.send({ type: 'append-segment', data: stream, meta: op.meta }); - await waitForIdle(sourceBufferActor.snapshot, taskSignal); + // Relocation (spike): on the FIRST media segment, fetch the whole body, + // read its tfdt baseMediaDecodeTime, pair with the init timescale to get + // this track's decode-time origin, and carry `timestampOffset = −origin` + // in the meta so the actor sets it before appending. Established once; + // subsequent segments stream as usual (the offset persists on the buffer). + if (relocation.enabled && !relocation.established) { + const bytes = await collect(await fetchBytes(op, { signal: taskSignal, minChunkSize: Infinity })); + const baseMediaDecodeTime = readFirstBaseMediaDecodeTime(bytes); + if (baseMediaDecodeTime !== undefined && relocation.timescale) { + relocation.offset = -(baseMediaDecodeTime / relocation.timescale); + // Publish on the actor snapshot so the text path can read the same + // offset for cue rebasing (see SegmentLoaderActorContext.relocationOffset). + setContext({ ...getContext(), relocationOffset: relocation.offset }); + } + relocation.established = true; + if (!taskSignal.aborted) { + const meta = relocation.offset !== undefined ? { ...op.meta, timestampOffset: relocation.offset } : op.meta; + sourceBufferActor.send({ type: 'append-segment', data: toArrayBuffer(bytes), meta }); + await waitForIdle(sourceBufferActor.snapshot, taskSignal); + } + } else { + const stream = await fetchBytes(op, { signal: taskSignal }); + if (!taskSignal.aborted) { + sourceBufferActor.send({ type: 'append-segment', data: stream, meta: op.meta }); + await waitForIdle(sourceBufferActor.snapshot, taskSignal); + } } } finally { setContext({ ...getContext(), inFlightSegmentId: null }); @@ -263,6 +348,8 @@ export function createSegmentLoaderActor( const forwardBufferConfig: ForwardBufferConfig = { ...DEFAULT_FORWARD_BUFFER_CONFIG, ...config.forwardBuffer }; const backBufferConfig: BackBufferConfig = { ...DEFAULT_BACK_BUFFER_CONFIG, ...config.backBuffer }; + // Per-track relocation state (spike), established once on the first media segment. + const relocation: RelocationState = { enabled: config.relocateTimestampOrigin ?? false, established: false }; const getBufferedSegments = (allSegments: readonly Segment[]): Segment[] => { // Exclude partial segments — they are still being streamed and must not be @@ -441,7 +528,7 @@ export function createSegmentLoaderActor( const scheduleAll = (tasks: LoadTask[], { getContext, setContext, runner }: Ctx): void => { tasks.forEach((op) => { runner - .schedule(makeLoadTask(op, { getContext, setContext, fetchBytes, sourceBufferActor })) + .schedule(makeLoadTask(op, { getContext, setContext, fetchBytes, sourceBufferActor, relocation })) .then(undefined, (e: unknown) => { if (e instanceof Error && e.name === 'AbortError') return; // On unexpected fetch/append errors, abort remaining tasks so a failed @@ -455,7 +542,7 @@ export function createSegmentLoaderActor( return createMachineActor SerialRunner>({ runner: () => new SerialRunner(), initial: 'idle', - context: { inFlightInitTrackId: null, inFlightSegmentId: null }, + context: { inFlightInitTrackId: null, inFlightSegmentId: null, relocationOffset: null }, states: { idle: { on: { diff --git a/packages/spf/src/playback/actors/dom/source-buffer.ts b/packages/spf/src/playback/actors/dom/source-buffer.ts index 59dcd976..e0658ff6 100644 --- a/packages/spf/src/playback/actors/dom/source-buffer.ts +++ b/packages/spf/src/playback/actors/dom/source-buffer.ts @@ -17,6 +17,13 @@ export type AppendSegmentMeta = Pick & trackId: Track['id']; /** Declared track bandwidth in bps (from playlist BANDWIDTH attribute). */ trackBandwidth?: number; + /** + * Non-zero-PTS relocation (spike): when present, set as + * `SourceBuffer.timestampOffset` before this append so native PTS is relocated + * onto a 0-based presentation timeline. Constant per source; the loader + * includes it once (on the first media segment). Absent = no relocation. + */ + timestampOffset?: number; }; export type { AppendData }; @@ -151,6 +158,12 @@ function appendSegmentTask( }); } + // Relocation (non-zero-PTS spike): set the offset before the coded frames + // are appended. The SerialRunner guarantees the buffer is idle here, so the + // assignment is safe; it's constant per source (loader sends it once). + if (meta.timestampOffset != null) { + sourceBuffer.timestampOffset = meta.timestampOffset; + } await appendSegment(sourceBuffer, message.data, taskSignal); // No abort check here: the physical SourceBuffer has been modified, so // the model must be updated to match regardless of signal state. diff --git a/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts b/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts index c536d318..d1f4bc7b 100644 --- a/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts +++ b/packages/spf/src/playback/behaviors/dom/setup-buffer-actors.ts @@ -146,7 +146,7 @@ function setupBufferActors { - const { type, selectedKey, actorKey, loaderKey, fetch, forwardBuffer, backBuffer } = config; + const { type, selectedKey, actorKey, loaderKey, fetch, forwardBuffer, backBuffer, relocateTimestampOrigin } = config; const derivedStateSignal = computed(() => { if (!context.mediaSource.get()) return 'preconditions-unmet'; const selection: TrackSelectionState = { @@ -172,7 +172,11 @@ function setupBufferActors | undefined; textTrackSegmentLoaderActor?: TextTrackSegmentLoaderActor | undefined; + /** + * Read-only, for cue rebasing under non-zero-PTS relocation (spike). The video + * loader establishes the shared decode-time offset and publishes it on its + * snapshot; the relocating resolver below reads it so text cues land on the + * same 0-based timeline as the relocated A/V. + */ + videoSegmentLoaderActor?: SegmentLoaderActor | undefined; } export interface TextTrackActorsConfig extends TextTrackSegmentLoaderActorConfig { resolveTextTrackSegment: TextTrackSegmentResolver; + /** + * Non-zero-PTS relocation (spike). When on, cues are rebased onto the relocated + * 0-based presentation timeline: `cueFinal = cueNative + timestampOffset`, where + * `cueNative = LOCAL + MPEGTS/90000` (`X-TIMESTAMP-MAP`) or the absolute cue time + * (no map), and `timestampOffset` is the video loader's established relocation + * offset. Off → the injected resolver is used unchanged (Tier 0). + */ + relocateTimestampOrigin?: boolean; +} + +/** + * Resolve when the video loader has published its relocation offset (spike). Text + * cues can't self-derive the origin for map-less sources (Mux), so the first text + * segment must wait for the A/V ground truth rather than land ~60s off. + */ +function awaitRelocationOffset(videoLoader: ReadonlySignal): Promise { + const read = (): number | null => { + const actor = peek(videoLoader); + return actor ? peek(actor.snapshot).context.relocationOffset : null; + }; + const current = read(); + if (current !== null) return Promise.resolve(current); + return new Promise((resolve) => { + let stop: (() => void) | undefined; + stop = effect(() => { + const actor = videoLoader.get(); + const offset = actor ? actor.snapshot.get().context.relocationOffset : null; + if (offset !== null) { + stop?.(); + resolve(offset); + } + }); + }); +} + +/** + * Wrap a cue resolver so cues are rebased onto the relocated 0-based timeline + * (spike). `X-TIMESTAMP-MAP` (Apple) puts cues at LOCAL time and the map's + * `mpegts/90000 − local` corrects them to the media timeline; absolute cues (Mux) + * need no correction. Both then shift by the video loader's relocation offset. + */ +function makeRelocatingResolver( + videoLoader: ReadonlySignal +): TextTrackSegmentResolver { + return async (url) => { + const { cues, metadata } = await resolveVttSegmentWithMetadata(url); + const offset = await awaitRelocationOffset(videoLoader); + const map = metadata.timestampMap; + const mapCorrection = map?.local !== undefined ? map.mpegts / 90000 - map.local : 0; + const delta = mapCorrection + offset; + if (delta !== 0) { + for (const cue of cues) { + cue.startTime += delta; + cue.endTime += delta; + } + } + return cues; + }; } function setupTextTrackActorsSetup({ @@ -48,6 +115,7 @@ function setupTextTrackActorsSetup({ mediaElement: ReadonlySignal; textTracksActor: Signal; textTrackSegmentLoaderActor: Signal; + videoSegmentLoaderActor: ReadonlySignal; }; config: TextTrackActorsConfig; }): () => void { @@ -55,12 +123,14 @@ function setupTextTrackActorsSetup({ const mediaElement = context.mediaElement.get(); if (!mediaElement) return; + const resolveTextTrackSegment = config.relocateTimestampOrigin + ? makeRelocatingResolver(context.videoSegmentLoaderActor) + : config.resolveTextTrackSegment; + const textTracksActor = createTextTracksActor(mediaElement); - const textTrackSegmentLoaderActor = createTextTrackSegmentLoaderActor( - textTracksActor, - config.resolveTextTrackSegment, - { forwardBuffer: config.forwardBuffer } - ); + const textTrackSegmentLoaderActor = createTextTrackSegmentLoaderActor(textTracksActor, resolveTextTrackSegment, { + forwardBuffer: config.forwardBuffer, + }); context.textTracksActor.set(textTracksActor); context.textTrackSegmentLoaderActor.set(textTrackSegmentLoaderActor); @@ -75,6 +145,6 @@ function setupTextTrackActorsSetup({ export const setupTextTrackActors = defineBehavior({ stateKeys: [], - contextKeys: ['mediaElement', 'textTracksActor', 'textTrackSegmentLoaderActor'], + contextKeys: ['mediaElement', 'textTracksActor', 'textTrackSegmentLoaderActor', 'videoSegmentLoaderActor'], setup: setupTextTrackActorsSetup, }); diff --git a/packages/spf/src/playback/behaviors/dom/tests/load-text-track-segments.test.ts b/packages/spf/src/playback/behaviors/dom/tests/load-text-track-segments.test.ts index 93e2c194..c3e43980 100644 --- a/packages/spf/src/playback/behaviors/dom/tests/load-text-track-segments.test.ts +++ b/packages/spf/src/playback/behaviors/dom/tests/load-text-track-segments.test.ts @@ -10,6 +10,7 @@ import type { Segment, TextTrack, } from '../../../../media/types'; +import type { SegmentLoaderActor } from '../../../actors/dom/segment-loader'; import type { TextTrackSegmentLoaderActor } from '../../../actors/text-track-segment-loader'; import type { TextTracksActor } from '../../../actors/text-tracks'; import { loadTextTrackSegments } from '../load-segments'; @@ -44,6 +45,9 @@ vi.mock('../../../../media/dom/text/resolve-vtt-segment', () => ({ } return Promise.resolve([new VTTCue(0, 5, `Subtitle from ${url}`)]); }), + resolveVttSegmentWithMetadata: vi.fn((url: string) => + Promise.resolve({ cues: [new VTTCue(0, 5, `Subtitle from ${url}`)], metadata: {} }) + ), destroyVttResolver: vi.fn(), })); @@ -66,6 +70,7 @@ function makeContext(initial: ComposedContext = {}): ContextSignals | undefined ) as ContextSignals['textTracksActor'], textTrackSegmentLoaderActor: signal(initial.textTrackSegmentLoaderActor), + videoSegmentLoaderActor: signal(initial.videoSegmentLoaderActor), }; } diff --git a/packages/spf/src/playback/engines/hls/engine.ts b/packages/spf/src/playback/engines/hls/engine.ts index e9df83b2..768f3a1d 100644 --- a/packages/spf/src/playback/engines/hls/engine.ts +++ b/packages/spf/src/playback/engines/hls/engine.ts @@ -274,6 +274,16 @@ export interface SimpleHlsEngineConfig extends ShareSignalsConfig