feat(spf): cross-track PDT timeline alignment

Add Track.startDate — the wall-clock time (epoch seconds) at a track's timeline
origin, derived in the parser as programDateTime − startTime and stable as the
window slides. Provisional from the manifest (refined later from the buffer).

Add alignTrackTimelines(tracks): a pure model transform that re-bases demuxed
tracks onto a common wall-clock origin by their startDate delta, so segments
with equal programDateTime get equal startTime — the cross-track A/V sync step.
Verified against the real demuxed Mux CMAF fixtures, where the 2s audio/video
skew resolves.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

packages/spf/src/media/align-track-timelines.ts

@@ -0,0 +1,44 @@
+import { isUndefined } from '@videojs/utils/predicate';
+import type { Track } from './types';
+
+/**
+ * Align multiple tracks onto a common wall-clock timeline so the same real
+ * instant has the same `startTime` across tracks — the cross-track A/V sync
+ * step for demuxed audio/video.
+ *
+ * Each track is parsed independently with its own origin, so a given wall-clock
+ * moment lands at different per-track `startTime`s (demuxed audio commonly
+ * starts a segment later than video). Every track carries `startDate` — the
+ * wall clock at its origin — so the skew between tracks is exactly the
+ * difference in their `startDate`s. This shifts each track's `startTime`s (and
+ * its origin `startTime`/`startDate`) by that difference, re-basing all tracks
+ * to the earliest origin. After alignment, segments with equal
+ * `programDateTime` have equal `startTime`.
+ *
+ * Tracks without a `startDate` (no `programDateTime` in the source) can't be
+ * aligned and pass through unchanged. The common origin is the earliest
+ * `startDate`, so no `startTime` goes negative.
+ */
+export function alignTrackTimelines<Tracks extends Track>(tracks: Tracks[]): Tracks[] {
+  const origins = tracks.map((track) => track.startDate).filter((date): date is number => !isUndefined(date));
+  if (origins.length < 2) {
+    return tracks;
+  }
+
+  const commonStartDate = Math.min(...origins);
+  return tracks.map((track) => {
+    if (isUndefined(track.startDate)) {
+      return track;
+    }
+    const shift = track.startDate - commonStartDate;
+    if (shift === 0) {
+      return track;
+    }
+    return {
+      ...track,
+      startTime: track.startTime + shift,
+      startDate: commonStartDate,
+      segments: track.segments.map((segment) => ({ ...segment, startTime: segment.startTime + shift })),
+    };
+  });
+}

packages/spf/src/media/hls/parse-media-playlist.ts

@@ -282,6 +282,16 @@ export function parseMediaPlaylist<T extends PartiallyResolvedTrack>(
     ? placeOnPreviousTimeline(previous, segments, mediaSequence, targetDuration)
     : { segments, startTime: 0 };
 
+  // Wall-clock anchor: `programDateTime − startTime` for the first PDT-bearing
+  // segment (constant along a linear timeline). Maps this track's origin to
+  // wall clock; recomputed each parse, so it stays stable as the window slides
+  // and is comparable across tracks for A/V alignment.
+  const anchorSegment = placed.segments.find((segment) => !isUndefined(segment.programDateTime));
+  const startDate =
+    anchorSegment && !isUndefined(anchorSegment.programDateTime)
+      ? anchorSegment.programDateTime - anchorSegment.startTime
+      : undefined;
+
   // Build initialization (VTT may not have init segment)
   const initialization =
     previous.type === 'text' && !initSegmentUrl
@@ -305,6 +315,7 @@ export function parseMediaPlaylist<T extends PartiallyResolvedTrack>(
     ...previous,
     mimeType,
     startTime: placed.startTime,
+    startDate,
     duration: trackDuration,
     segments: placed.segments,
     initialization,

packages/spf/src/media/hls/tests/parse-media-playlist.test.ts

@@ -642,6 +642,52 @@ s1.ts`;
       const r = parseMediaPlaylist(text, videoShell);
       expect(r.segments.every((s) => s.programDateTime === undefined)).toBe(true);
     });
+
+    it('exposes Track.startDate as the wall-clock at the origin (programDateTime − startTime)', () => {
+      const text = `#EXTM3U
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:10.000Z
+#EXTINF:4,
+s0.ts
+#EXTINF:4,
+s1.ts`;
+      // First parse anchors startTime at 0, so the origin maps to s0's wall clock.
+      expect(parseMediaPlaylist(text, videoShell).startDate).toBe(epoch('2026-01-01T00:00:10.000Z'));
+    });
+
+    it('keeps Track.startDate stable as the window slides', () => {
+      const first = `#EXTM3U
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:00.000Z
+#EXTINF:4,
+s0.ts
+#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:04.000Z
+#EXTINF:4,
+s1.ts`;
+      const prev = parseMediaPlaylist(first, videoShell);
+      expect(prev.startDate).toBe(epoch('2026-01-01T00:00:00.000Z'));
+
+      // Window slid by one: s0 rolled off, s1 is now first (startTime carried to 4).
+      const reload = `#EXTM3U
+#EXT-X-MEDIA-SEQUENCE:1
+#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:04.000Z
+#EXTINF:4,
+s1.ts
+#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:08.000Z
+#EXTINF:4,
+s2.ts`;
+      const next = parseMediaPlaylist(reload, prev);
+      expect(next.segments[0]?.startTime).toBe(4); // window advanced
+      expect(next.startDate).toBe(epoch('2026-01-01T00:00:00.000Z')); // origin unchanged
+    });
+
+    it('leaves Track.startDate undefined when the source carries no PDT', () => {
+      const text = `#EXTM3U
+#EXT-X-MEDIA-SEQUENCE:0
+#EXTINF:4,
+s0.ts`;
+      expect(parseMediaPlaylist(text, videoShell).startDate).toBeUndefined();
+    });
   });
 
   describe('real Mux live snapshots (fixtures)', () => {
@@ -717,5 +763,17 @@ s1.ts`;
       expect(v82?.startTime).toBe(2);
       expect(a82?.startTime).toBe(0);
     });
+
+    it('exposes per-track startDate whose audio/video delta is the relative skew', () => {
+      const video = parseMediaPlaylist(liveCmafVideo, videoShell);
+      const audio = parseMediaPlaylist(liveCmafAudio, audioShell);
+
+      expect(video.startDate).toBeDefined();
+      expect(audio.startDate).toBeDefined();
+      // Each track's origin (startTime 0) sits at a different real instant —
+      // audio's window starts one 2s segment later — so the startDate delta is
+      // the relative A/V skew a cross-track aligner removes.
+      expect((audio.startDate ?? 0) - (video.startDate ?? 0)).toBeCloseTo(2, 3);
+    });
   });
 });

packages/spf/src/media/tests/align-track-timelines.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, it } from 'vitest';
+import { alignTrackTimelines } from '../align-track-timelines';
+import { parseMediaPlaylist } from '../hls/parse-media-playlist';
+import liveCmafAudio from '../hls/tests/fixtures/live-cmaf-audio.m3u8?raw';
+import liveCmafVideo from '../hls/tests/fixtures/live-cmaf-video.m3u8?raw';
+import type { PartiallyResolvedAudioTrack, PartiallyResolvedVideoTrack, Segment, Track } from '../types';
+
+function makeTrack(startDate: number | undefined, segments: Array<{ startTime: number; pdt?: number }>): Track {
+  return {
+    type: 'video',
+    id: 'track',
+    url: 'https://example.com/playlist.m3u8',
+    mimeType: 'video/mp4',
+    bandwidth: 0,
+    duration: Number.POSITIVE_INFINITY,
+    startTime: segments[0]?.startTime ?? 0,
+    startDate,
+    segments: segments.map(
+      (s, i): Segment => ({
+        id: `segment-${i}`,
+        url: `s${i}.m4s`,
+        duration: 2,
+        startTime: s.startTime,
+        ...(s.pdt === undefined ? {} : { programDateTime: s.pdt }),
+      })
+    ),
+  };
+}
+
+describe('alignTrackTimelines', () => {
+  it('re-bases tracks to the earliest origin so equal PDT yields equal startTime', () => {
+    // video origin at wall clock 1000; audio origin 2s later (starts a segment behind).
+    const video = makeTrack(1000, [
+      { startTime: 0, pdt: 1000 },
+      { startTime: 2, pdt: 1002 },
+      { startTime: 4, pdt: 1004 },
+    ]);
+    const audio = makeTrack(1002, [
+      { startTime: 0, pdt: 1002 },
+      { startTime: 2, pdt: 1004 },
+    ]);
+
+    const [alignedVideo, alignedAudio] = alignTrackTimelines([video, audio]);
+
+    // Common origin = earliest startDate (video's, 1000).
+    expect(alignedVideo?.startDate).toBe(1000);
+    expect(alignedAudio?.startDate).toBe(1000);
+    // Video unchanged (it is the earliest); audio shifted forward 2s.
+    expect(alignedVideo?.segments.map((s) => s.startTime)).toEqual([0, 2, 4]);
+    expect(alignedAudio?.segments.map((s) => s.startTime)).toEqual([2, 4]);
+
+    // The same instant (PDT 1004) now has the same startTime in both tracks.
+    const vAt1004 = alignedVideo?.segments.find((s) => s.programDateTime === 1004);
+    const aAt1004 = alignedAudio?.segments.find((s) => s.programDateTime === 1004);
+    expect(vAt1004?.startTime).toBe(aAt1004?.startTime);
+  });
+
+  it('passes through when fewer than two tracks carry a startDate', () => {
+    const dated = makeTrack(1000, [{ startTime: 0, pdt: 1000 }]);
+    const undated = makeTrack(undefined, [{ startTime: 0 }]);
+
+    expect(alignTrackTimelines([dated])).toEqual([dated]);
+    // Only one track has a date → nothing to align against.
+    const result = alignTrackTimelines([dated, undated]);
+    expect(result[0]).toBe(dated);
+    expect(result[1]).toBe(undated);
+  });
+
+  it('leaves already-aligned tracks untouched (no shift)', () => {
+    const a = makeTrack(1000, [{ startTime: 0, pdt: 1000 }]);
+    const b = makeTrack(1000, [{ startTime: 0, pdt: 1000 }]);
+    const [ra, rb] = alignTrackTimelines([a, b]);
+    expect(ra).toBe(a); // identity preserved when shift === 0
+    expect(rb).toBe(b);
+  });
+
+  it('aligns the real demuxed Mux CMAF audio/video (resolves the 2s skew)', () => {
+    const videoShell: PartiallyResolvedVideoTrack = {
+      type: 'video',
+      id: 'video-0',
+      url: 'https://example.com/video/playlist.m3u8',
+      bandwidth: 2191200,
+      width: 1280,
+      height: 572,
+      codecs: ['avc1.640020'],
+      frameRate: { frameRateNumerator: 30 },
+      mimeType: 'video/mp4',
+    };
+    const audioShell: PartiallyResolvedAudioTrack = {
+      type: 'audio',
+      id: 'audio-hi-0',
+      url: 'https://example.com/audio/playlist.m3u8',
+      groupId: 'audio-hi-0',
+      name: 'Default',
+      language: 'und',
+      codecs: ['mp4a.40.2'],
+      mimeType: 'audio/mp4',
+      bandwidth: 0,
+      sampleRate: 48000,
+      channels: 2,
+    };
+
+    const [video, audio] = alignTrackTimelines([
+      parseMediaPlaylist(liveCmafVideo, videoShell),
+      parseMediaPlaylist(liveCmafAudio, audioShell),
+    ]);
+
+    const v82 = video?.segments.find((s) => s.id === 'segment-82');
+    const a82 = audio?.segments.find((s) => s.id === 'segment-82');
+    // Before alignment these disagreed (2 vs 0); after, the same instant matches.
+    expect(v82?.startTime).toBe(a82?.startTime);
+    expect(video?.startDate).toBe(audio?.startDate);
+  });
+});

packages/spf/src/media/types/index.ts

@@ -138,6 +138,20 @@ export type Track = Ham &
     bandwidth: number;
     initialization?: AddressableObject;
     segments: Segment[];
+    /**
+     * Wall-clock time (epoch seconds) corresponding to the track's timeline
+     * origin (`startTime`) — i.e. `programDateTime − startTime`, the single
+     * rolling anchor that maps this track's media timeline to wall clock.
+     * Optional: absent when no segment carries `programDateTime`.
+     *
+     * Provisional from the manifest, where the origin is the first fetched
+     * segment; later refined from the buffer (`buffered`/`tfdt`) to pin the
+     * origin to encoded-media zero. Comparable across tracks: the difference in
+     * `startDate` between demuxed audio and video is their relative skew — the
+     * offset a cross-track aligner removes — and equal `programDateTime` across
+     * tracks marks the same presentation instant.
+     */
+    startDate?: number;
   };
 
 /**