diff --git a/packages/spf/src/media/align-track-timelines.ts b/packages/spf/src/media/align-track-timelines.ts new file mode 100644 index 00000000..217c2140 --- /dev/null +++ b/packages/spf/src/media/align-track-timelines.ts @@ -0,0 +1,44 @@ +import { isUndefined } from '@videojs/utils/predicate'; +import type { Track } from './types'; + +/** + * Align multiple tracks onto a common wall-clock timeline so the same real + * instant has the same `startTime` across tracks — the cross-track A/V sync + * step for demuxed audio/video. + * + * Each track is parsed independently with its own origin, so a given wall-clock + * moment lands at different per-track `startTime`s (demuxed audio commonly + * starts a segment later than video). Every track carries `startDate` — the + * wall clock at its origin — so the skew between tracks is exactly the + * difference in their `startDate`s. This shifts each track's `startTime`s (and + * its origin `startTime`/`startDate`) by that difference, re-basing all tracks + * to the earliest origin. After alignment, segments with equal + * `programDateTime` have equal `startTime`. + * + * Tracks without a `startDate` (no `programDateTime` in the source) can't be + * aligned and pass through unchanged. The common origin is the earliest + * `startDate`, so no `startTime` goes negative. + */ +export function alignTrackTimelines(tracks: Tracks[]): Tracks[] { + const origins = tracks.map((track) => track.startDate).filter((date): date is number => !isUndefined(date)); + if (origins.length < 2) { + return tracks; + } + + const commonStartDate = Math.min(...origins); + return tracks.map((track) => { + if (isUndefined(track.startDate)) { + return track; + } + const shift = track.startDate - commonStartDate; + if (shift === 0) { + return track; + } + return { + ...track, + startTime: track.startTime + shift, + startDate: commonStartDate, + segments: track.segments.map((segment) => ({ ...segment, startTime: segment.startTime + shift })), + }; + }); +} diff --git a/packages/spf/src/media/hls/parse-media-playlist.ts b/packages/spf/src/media/hls/parse-media-playlist.ts index cd94b8ed..87ecb526 100644 --- a/packages/spf/src/media/hls/parse-media-playlist.ts +++ b/packages/spf/src/media/hls/parse-media-playlist.ts @@ -282,6 +282,16 @@ export function parseMediaPlaylist( ? placeOnPreviousTimeline(previous, segments, mediaSequence, targetDuration) : { segments, startTime: 0 }; + // Wall-clock anchor: `programDateTime − startTime` for the first PDT-bearing + // segment (constant along a linear timeline). Maps this track's origin to + // wall clock; recomputed each parse, so it stays stable as the window slides + // and is comparable across tracks for A/V alignment. + const anchorSegment = placed.segments.find((segment) => !isUndefined(segment.programDateTime)); + const startDate = + anchorSegment && !isUndefined(anchorSegment.programDateTime) + ? anchorSegment.programDateTime - anchorSegment.startTime + : undefined; + // Build initialization (VTT may not have init segment) const initialization = previous.type === 'text' && !initSegmentUrl @@ -305,6 +315,7 @@ export function parseMediaPlaylist( ...previous, mimeType, startTime: placed.startTime, + startDate, duration: trackDuration, segments: placed.segments, initialization, diff --git a/packages/spf/src/media/hls/tests/parse-media-playlist.test.ts b/packages/spf/src/media/hls/tests/parse-media-playlist.test.ts index a82fb43d..ba21ad8e 100644 --- a/packages/spf/src/media/hls/tests/parse-media-playlist.test.ts +++ b/packages/spf/src/media/hls/tests/parse-media-playlist.test.ts @@ -642,6 +642,52 @@ s1.ts`; const r = parseMediaPlaylist(text, videoShell); expect(r.segments.every((s) => s.programDateTime === undefined)).toBe(true); }); + + it('exposes Track.startDate as the wall-clock at the origin (programDateTime − startTime)', () => { + const text = `#EXTM3U +#EXT-X-MEDIA-SEQUENCE:0 +#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:10.000Z +#EXTINF:4, +s0.ts +#EXTINF:4, +s1.ts`; + // First parse anchors startTime at 0, so the origin maps to s0's wall clock. + expect(parseMediaPlaylist(text, videoShell).startDate).toBe(epoch('2026-01-01T00:00:10.000Z')); + }); + + it('keeps Track.startDate stable as the window slides', () => { + const first = `#EXTM3U +#EXT-X-MEDIA-SEQUENCE:0 +#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:00.000Z +#EXTINF:4, +s0.ts +#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:04.000Z +#EXTINF:4, +s1.ts`; + const prev = parseMediaPlaylist(first, videoShell); + expect(prev.startDate).toBe(epoch('2026-01-01T00:00:00.000Z')); + + // Window slid by one: s0 rolled off, s1 is now first (startTime carried to 4). + const reload = `#EXTM3U +#EXT-X-MEDIA-SEQUENCE:1 +#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:04.000Z +#EXTINF:4, +s1.ts +#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:08.000Z +#EXTINF:4, +s2.ts`; + const next = parseMediaPlaylist(reload, prev); + expect(next.segments[0]?.startTime).toBe(4); // window advanced + expect(next.startDate).toBe(epoch('2026-01-01T00:00:00.000Z')); // origin unchanged + }); + + it('leaves Track.startDate undefined when the source carries no PDT', () => { + const text = `#EXTM3U +#EXT-X-MEDIA-SEQUENCE:0 +#EXTINF:4, +s0.ts`; + expect(parseMediaPlaylist(text, videoShell).startDate).toBeUndefined(); + }); }); describe('real Mux live snapshots (fixtures)', () => { @@ -717,5 +763,17 @@ s1.ts`; expect(v82?.startTime).toBe(2); expect(a82?.startTime).toBe(0); }); + + it('exposes per-track startDate whose audio/video delta is the relative skew', () => { + const video = parseMediaPlaylist(liveCmafVideo, videoShell); + const audio = parseMediaPlaylist(liveCmafAudio, audioShell); + + expect(video.startDate).toBeDefined(); + expect(audio.startDate).toBeDefined(); + // Each track's origin (startTime 0) sits at a different real instant — + // audio's window starts one 2s segment later — so the startDate delta is + // the relative A/V skew a cross-track aligner removes. + expect((audio.startDate ?? 0) - (video.startDate ?? 0)).toBeCloseTo(2, 3); + }); }); }); diff --git a/packages/spf/src/media/tests/align-track-timelines.test.ts b/packages/spf/src/media/tests/align-track-timelines.test.ts new file mode 100644 index 00000000..a392f524 --- /dev/null +++ b/packages/spf/src/media/tests/align-track-timelines.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; +import { alignTrackTimelines } from '../align-track-timelines'; +import { parseMediaPlaylist } from '../hls/parse-media-playlist'; +import liveCmafAudio from '../hls/tests/fixtures/live-cmaf-audio.m3u8?raw'; +import liveCmafVideo from '../hls/tests/fixtures/live-cmaf-video.m3u8?raw'; +import type { PartiallyResolvedAudioTrack, PartiallyResolvedVideoTrack, Segment, Track } from '../types'; + +function makeTrack(startDate: number | undefined, segments: Array<{ startTime: number; pdt?: number }>): Track { + return { + type: 'video', + id: 'track', + url: 'https://example.com/playlist.m3u8', + mimeType: 'video/mp4', + bandwidth: 0, + duration: Number.POSITIVE_INFINITY, + startTime: segments[0]?.startTime ?? 0, + startDate, + segments: segments.map( + (s, i): Segment => ({ + id: `segment-${i}`, + url: `s${i}.m4s`, + duration: 2, + startTime: s.startTime, + ...(s.pdt === undefined ? {} : { programDateTime: s.pdt }), + }) + ), + }; +} + +describe('alignTrackTimelines', () => { + it('re-bases tracks to the earliest origin so equal PDT yields equal startTime', () => { + // video origin at wall clock 1000; audio origin 2s later (starts a segment behind). + const video = makeTrack(1000, [ + { startTime: 0, pdt: 1000 }, + { startTime: 2, pdt: 1002 }, + { startTime: 4, pdt: 1004 }, + ]); + const audio = makeTrack(1002, [ + { startTime: 0, pdt: 1002 }, + { startTime: 2, pdt: 1004 }, + ]); + + const [alignedVideo, alignedAudio] = alignTrackTimelines([video, audio]); + + // Common origin = earliest startDate (video's, 1000). + expect(alignedVideo?.startDate).toBe(1000); + expect(alignedAudio?.startDate).toBe(1000); + // Video unchanged (it is the earliest); audio shifted forward 2s. + expect(alignedVideo?.segments.map((s) => s.startTime)).toEqual([0, 2, 4]); + expect(alignedAudio?.segments.map((s) => s.startTime)).toEqual([2, 4]); + + // The same instant (PDT 1004) now has the same startTime in both tracks. + const vAt1004 = alignedVideo?.segments.find((s) => s.programDateTime === 1004); + const aAt1004 = alignedAudio?.segments.find((s) => s.programDateTime === 1004); + expect(vAt1004?.startTime).toBe(aAt1004?.startTime); + }); + + it('passes through when fewer than two tracks carry a startDate', () => { + const dated = makeTrack(1000, [{ startTime: 0, pdt: 1000 }]); + const undated = makeTrack(undefined, [{ startTime: 0 }]); + + expect(alignTrackTimelines([dated])).toEqual([dated]); + // Only one track has a date → nothing to align against. + const result = alignTrackTimelines([dated, undated]); + expect(result[0]).toBe(dated); + expect(result[1]).toBe(undated); + }); + + it('leaves already-aligned tracks untouched (no shift)', () => { + const a = makeTrack(1000, [{ startTime: 0, pdt: 1000 }]); + const b = makeTrack(1000, [{ startTime: 0, pdt: 1000 }]); + const [ra, rb] = alignTrackTimelines([a, b]); + expect(ra).toBe(a); // identity preserved when shift === 0 + expect(rb).toBe(b); + }); + + it('aligns the real demuxed Mux CMAF audio/video (resolves the 2s skew)', () => { + const videoShell: PartiallyResolvedVideoTrack = { + type: 'video', + id: 'video-0', + url: 'https://example.com/video/playlist.m3u8', + bandwidth: 2191200, + width: 1280, + height: 572, + codecs: ['avc1.640020'], + frameRate: { frameRateNumerator: 30 }, + mimeType: 'video/mp4', + }; + const audioShell: PartiallyResolvedAudioTrack = { + type: 'audio', + id: 'audio-hi-0', + url: 'https://example.com/audio/playlist.m3u8', + groupId: 'audio-hi-0', + name: 'Default', + language: 'und', + codecs: ['mp4a.40.2'], + mimeType: 'audio/mp4', + bandwidth: 0, + sampleRate: 48000, + channels: 2, + }; + + const [video, audio] = alignTrackTimelines([ + parseMediaPlaylist(liveCmafVideo, videoShell), + parseMediaPlaylist(liveCmafAudio, audioShell), + ]); + + const v82 = video?.segments.find((s) => s.id === 'segment-82'); + const a82 = audio?.segments.find((s) => s.id === 'segment-82'); + // Before alignment these disagreed (2 vs 0); after, the same instant matches. + expect(v82?.startTime).toBe(a82?.startTime); + expect(video?.startDate).toBe(audio?.startDate); + }); +}); diff --git a/packages/spf/src/media/types/index.ts b/packages/spf/src/media/types/index.ts index 03ff6bd5..230012f8 100644 --- a/packages/spf/src/media/types/index.ts +++ b/packages/spf/src/media/types/index.ts @@ -138,6 +138,20 @@ export type Track = Ham & bandwidth: number; initialization?: AddressableObject; segments: Segment[]; + /** + * Wall-clock time (epoch seconds) corresponding to the track's timeline + * origin (`startTime`) — i.e. `programDateTime − startTime`, the single + * rolling anchor that maps this track's media timeline to wall clock. + * Optional: absent when no segment carries `programDateTime`. + * + * Provisional from the manifest, where the origin is the first fetched + * segment; later refined from the buffer (`buffered`/`tfdt`) to pin the + * origin to encoded-media zero. Comparable across tracks: the difference in + * `startDate` between demuxed audio and video is their relative skew — the + * offset a cross-track aligner removes — and equal `programDateTime` across + * tracks marks the same presentation instant. + */ + startDate?: number; }; /**