Merge pull request #4303 from cjpillsbury/bugfix/live-start-bugs

Resolve edge case issues in initial load for live content, which are more likely with LL-HLS:
- Accounts for maxFragLookUpTolerance when determining start position (not just maxBufferHole)
- Accounts for the possibility of incongruous segments on initial load of audio tracks when applying sliding/pdt and instead attempts to align based on Pantos §6.2.4. 

Providing Variant Streams (https://datatracker.ietf.org/doc/html/draft-pantos-http-live-streaming#section-6.2.4)

"If any Media Playlist in a Master Playlist contains an EXT-X-PROGRAM-DATE-TIME tag, then all Media Playlists in that Master Playlist MUST contain EXT-X-PROGRAM-DATE-TIME tags with consistent mappings of date and time to media timestamps."

src/controller/audio-stream-controller.ts

@@ -9,7 +9,7 @@ import ChunkCache from '../demux/chunk-cache';
 import TransmuxerInterface from '../demux/transmuxer-interface';
 import { ChunkMetadata } from '../types/transmuxer';
 import { fragmentWithinToleranceTest } from './fragment-finders';
-import { alignPDT } from '../utils/discontinuities';
+import { alignMediaPlaylistByPDT } from '../utils/discontinuities';
 import { ErrorDetails } from '../errors';
 import { logger } from '../utils/logger';
 import type { NetworkComponentAPI } from '../types/component-api';
@@ -144,6 +144,7 @@ class AudioStreamController
       this.startPosition =
       this.lastCurrentTime =
         startPosition;
+
     this.tick();
   }
 
@@ -440,7 +441,9 @@ class AudioStreamController
         newDetails.hasProgramDateTime &&
         mainDetails.hasProgramDateTime
       ) {
-        alignPDT(newDetails, mainDetails);
+        // Make sure our audio rendition is aligned with the "main" rendition, using
+        // pdt as our reference times.
+        alignMediaPlaylistByPDT(newDetails, mainDetails);
         sliding = newDetails.fragments[0].start;
       } else {
         sliding = this.alignPlaylists(newDetails, track.details);

src/controller/stream-controller.ts

@@ -980,7 +980,11 @@ export default class StreamController
       const buffered = BufferHelper.getBuffered(media);
       const bufferStart = buffered.length ? buffered.start(0) : 0;
       const delta = bufferStart - startPosition;
-      if (delta > 0 && delta < this.config.maxBufferHole) {
+      if (
+        delta > 0 &&
+        (delta < this.config.maxBufferHole ||
+          delta < this.config.maxFragLookUpTolerance)
+      ) {
         logger.log(
           `adjusting start position by ${delta} to match buffer start`
         );

src/controller/subtitle-stream-controller.ts

@@ -2,7 +2,7 @@ import { Events } from '../events';
 import { logger } from '../utils/logger';
 import { BufferHelper } from '../utils/buffer-helper';
 import { findFragmentByPDT, findFragmentByPTS } from './fragment-finders';
-import { alignPDT } from '../utils/discontinuities';
+import { alignMediaPlaylistByPDT } from '../utils/discontinuities';
 import { addSliding } from './level-helper';
 import { FragmentState } from './fragment-tracker';
 import BaseStreamController, { State } from './base-stream-controller';
@@ -251,7 +251,7 @@ export class SubtitleStreamController
       const mainSlidingStartFragment = mainDetails.fragments[0];
       if (!track.details) {
         if (newDetails.hasProgramDateTime && mainDetails.hasProgramDateTime) {
-          alignPDT(newDetails, mainDetails);
+          alignMediaPlaylistByPDT(newDetails, mainDetails);
         } else if (mainSlidingStartFragment) {
           // line up live playlist with main so that fragments in range are loaded
           addSliding(newDetails, mainSlidingStartFragment.start);

src/utils/discontinuities.ts

@@ -173,3 +173,56 @@ export function alignPDT(details: LevelDetails, lastDetails: LevelDetails) {
     adjustSlidingStart(sliding, details);
   }
 }
+
+export function alignFragmentByPDTDelta(frag: Fragment, delta: number) {
+  const { programDateTime } = frag;
+  if (!programDateTime) return;
+  const start = (programDateTime - delta) / 1000;
+  frag.start = frag.startPTS = start;
+  frag.endPTS = start + frag.duration;
+}
+
+/**
+ * Ensures appropriate time-alignment between renditions based on PDT. Unlike `alignPDT`, which adjusts
+ * the timeline based on the delta between PDTs of the 0th fragment of two playlists/`LevelDetails`,
+ * this function assumes the timelines represented in `refDetails` are accurate, including the PDTs,
+ * and uses the "wallclock"/PDT timeline as a cross-reference to `details`, adjusting the presentation
+ * times/timelines of `details` accordingly.
+ * Given the asynchronous nature of fetches and initial loads of live `main` and audio/subtitle tracks,
+ * the primary purpose of this function is to ensure the "local timelines" of audio/subtitle tracks
+ * are aligned to the main/video timeline, using PDT as the cross-reference/"anchor" that should
+ * be consistent across playlists, per the HLS spec.
+ * @param details - The details of the rendition you'd like to time-align (e.g. an audio rendition).
+ * @param refDetails - The details of the reference rendition with start and PDT times for alignment.
+ */
+export function alignMediaPlaylistByPDT(
+  details: LevelDetails,
+  refDetails: LevelDetails
+) {
+  // This check protects the unsafe "!" usage below for null program date time access.
+  if (
+    !refDetails.fragments.length ||
+    !details.hasProgramDateTime ||
+    !refDetails.hasProgramDateTime
+  ) {
+    return;
+  }
+  const refPDT = refDetails.fragments[0].programDateTime!; // hasProgramDateTime check above makes this safe.
+  const refStart = refDetails.fragments[0].start;
+  // Use the delta between the reference details' presentation timeline's start time and its PDT
+  // to align the other rendtion's timeline.
+  const delta = refPDT - refStart * 1000;
+  // Per spec: "If any Media Playlist in a Master Playlist contains an EXT-X-PROGRAM-DATE-TIME tag, then all
+  // Media Playlists in that Master Playlist MUST contain EXT-X-PROGRAM-DATE-TIME tags with consistent mappings
+  // of date and time to media timestamps."
+  // So we should be able to use each rendition's PDT as a reference time and use the delta to compute our relevant
+  // start and end times.
+  // NOTE: This code assumes each level/details timelines have already been made "internally consistent"
+  details.fragments.forEach((frag) => {
+    alignFragmentByPDTDelta(frag, delta);
+  });
+  if (details.fragmentHint) {
+    alignFragmentByPDTDelta(details.fragmentHint, delta);
+  }
+  details.alignedSliding = true;
+}

tests/unit/utils/discontinuities.js

@@ -3,6 +3,7 @@ import {
   findDiscontinuousReferenceFrag,
   adjustSlidingStart,
   alignPDT,
+  alignMediaPlaylistByPDT,
 } from '../../../src/utils/discontinuities';
 
 const mockReferenceFrag = {
@@ -73,6 +74,106 @@ describe('level-helper', function () {
     expect(details.alignedSliding).to.be.true;
   });
 
+  it('aligns level fragments times based on PDT and start time of reference level details', function () {
+    const lastLevel = {
+      details: {
+        PTSKnown: false,
+        alignedSliding: false,
+        hasProgramDateTime: true,
+        fragments: [
+          {
+            start: 18,
+            startPTS: undefined,
+            endPTS: undefined,
+            duration: 2,
+            programDateTime: 1629821766107,
+          },
+          {
+            start: 20,
+            startPTS: undefined,
+            endPTS: 22,
+            duration: 2,
+            programDateTime: 1629821768107,
+          },
+          {
+            start: 22,
+            startPTS: 22,
+            endPTS: 30,
+            duration: 8,
+            programDateTime: 1629821770107,
+          },
+        ],
+        fragmentHint: {
+          start: 30,
+          startPTS: 30,
+          endPTS: 32,
+          duration: 2,
+          programDateTime: 1629821778107,
+        },
+      },
+    };
+
+    const refDetails = {
+      fragments: [
+        {
+          start: 18,
+          startPTS: undefined,
+          endPTS: undefined,
+          duration: 2,
+          programDateTime: 1629821768107,
+        },
+      ],
+      PTSKnown: false,
+      alignedSliding: false,
+      hasProgramDateTime: true,
+    };
+
+    const detailsExpected = {
+      fragments: [
+        {
+          start: 16,
+          startPTS: 16,
+          endPTS: 18,
+          duration: 2,
+          programDateTime: 1629821766107,
+        },
+        {
+          start: 18,
+          startPTS: 18,
+          endPTS: 20,
+          duration: 2,
+          programDateTime: 1629821768107,
+        },
+        {
+          start: 20,
+          startPTS: 20,
+          endPTS: 28,
+          duration: 8,
+          programDateTime: 1629821770107,
+        },
+      ],
+      fragmentHint: {
+        start: 28,
+        startPTS: 28,
+        endPTS: 30,
+        duration: 2,
+        programDateTime: 1629821778107,
+      },
+      PTSKnown: false,
+      alignedSliding: true,
+      hasProgramDateTime: true,
+    };
+    alignMediaPlaylistByPDT(lastLevel.details, refDetails);
+    expect(
+      lastLevel.details,
+      `actual:\n\n${JSON.stringify(
+        lastLevel.details,
+        null,
+        2
+      )}\n\nexpected\n\n${JSON.stringify(detailsExpected, null, 2)}`
+    ).to.deep.equal(detailsExpected);
+  });
+
   it('adjusts level fragments without overlapping CC range but with programDateTime info', function () {
     const lastLevel = {
       details: {