fix(hls): Fix AV sync issues, fallback to sequence numbers if PROGRAM…

…-DATE-TIME ignored (shaka-project#4289)

We now have an explicit fallback to sync on sequence numbers if
PROGRAM-DATE-TIME is explicitly ignored.  This is more robust than
relying on whatever happens to be first in the various media
playlists.

This also refactors how PROGRAM-DATE-TIME is used.  The date will now
be used to adjust segment reference start times, rather than
overriding the time used in StreamingEngine.  (This was hard to
discover when reading the HLS parser.)  Now all HLS sync logic is in
the HLS parser.

Closes shaka-project#4287

externs/shaka/player.js

@@ -777,8 +777,10 @@ shaka.extern.DashManifestConfiguration;
  *   <i>Defaults to <code>'avc1.42E01E'</code>.</i>
  * @property {boolean} ignoreManifestProgramDateTime
  *   If <code>true</code>, the HLS parser will ignore the
- *   <code>EXT-X-PROGRAM-DATE-TIME</code> tags in the manifest.
- *   Meant for tags that are incorrect or malformed.
+ *   <code>EXT-X-PROGRAM-DATE-TIME</code> tags in the manifest and use media
+ *   sequence numbers instead.
+ *   Meant for streams where <code>EXT-X-PROGRAM-DATE-TIME</code> is incorrect
+ *   or malformed.
  *   <i>Defaults to <code>false</code>.</i>
  * @property {string} mediaPlaylistFullMimeType
  *   A string containing a full mime type, including both the basic mime type

lib/hls/hls_parser.js

@@ -121,13 +121,19 @@ shaka.hls.HlsParser = class {
     this.updatePlaylistDelay_ = 0;
 
     /**
-     * A time offset to apply to EXT-X-PROGRAM-DATE-TIME values to normalize
-     * them so that they start at 0. This is necessary because these times will
-     * be used to set presentation times for segments.
-     * null means we don't have enough data yet.
+     * If true, we have already calculated offsets to synchronize streams.
+     * Offsets are computed in syncStreams*_().
+     * @private {boolean}
+     */
+    this.streamsSynced_ = false;
+
+    /**
+     * The minimum sequence number for generated segments, when ignoring
+     * EXT-X-PROGRAM-DATE-TIME.
+     *
      * @private {?number}
      */
-    this.syncTimeOffset_ = null;
+    this.minSequenceNumber_ = null;
 
     /**
      * This timer is used to trigger the start of a manifest update. A manifest
@@ -361,39 +367,155 @@ shaka.hls.HlsParser = class {
   }
 
   /**
-   * If necessary, makes sure that sync times will be normalized to 0, so that
-   * a stream does not start buffering at 50 years in because sync times are
-   * measured in time since 1970.
+   * Align all streams by sequence number by dropping early segments.  Then
+   * offset all streams to begin at presentation time 0.
    * @private
    */
-  calculateSyncTimeOffset_() {
-    if (this.syncTimeOffset_ != null) {
-      // The offset was already calculated.
+  syncStreamsWithSequenceNumber_() {
+    if (this.streamsSynced_) {
       return;
     }
 
-    const segments = new Set();
-    let lowestSyncTime = Infinity;
+    // Sync using media sequence number.  Find the highest starting sequence
+    // number among all streams.  Later, we will drop any references to
+    // earlier segments in other streams, then offset everything back to 0.
+    let highestStartingSequenceNumber = -1;
+    const firstSequenceNumberMap = new Map();
+
     for (const streamInfo of this.uriToStreamInfosMap_.values()) {
       const segmentIndex = streamInfo.stream.segmentIndex;
       if (segmentIndex) {
-        segmentIndex.forEachTopLevelReference((segment) => {
-          if (segment.syncTime != null) {
-            lowestSyncTime = Math.min(lowestSyncTime, segment.syncTime);
-            segments.add(segment);
+        const segment0 = segmentIndex.earliestReference();
+        if (segment0) {
+          // This looks inefficient, but iteration order is insertion order.
+          // So the very first entry should be the one we want.
+          // We assert that this holds true so that we are alerted by debug
+          // builds and tests if it changes.  We still do a loop, though, so
+          // that the code functions correctly in production no matter what.
+          if (goog.DEBUG) {
+            const firstSequenceStartTime =
+                streamInfo.mediaSequenceToStartTime.values().next().value;
+            goog.asserts.assert(
+                firstSequenceStartTime == segment0.startTime,
+                'Sequence number map is not ordered as expected!');
+          }
+          for (const [sequence, start] of streamInfo.mediaSequenceToStartTime) {
+            if (start == segment0.startTime) {
+              firstSequenceNumberMap.set(streamInfo, sequence);
+
+              highestStartingSequenceNumber = Math.max(
+                  highestStartingSequenceNumber, sequence);
+              break;
+            }
           }
-        });
+        }
+      }
+    }
+
+    if (highestStartingSequenceNumber < 0) {
+      // Nothing to sync.
+      return;
+    }
+
+    // From now on, updates will ignore any references before this number.
+    this.minSequenceNumber_ = highestStartingSequenceNumber;
+
+    shaka.log.debug('Syncing HLS streams against base sequence number:',
+        this.minSequenceNumber_);
+
+    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+      const segmentIndex = streamInfo.stream.segmentIndex;
+      if (segmentIndex) {
+        // Drop any earlier references.
+        const numSegmentsToDrop = this.minSequenceNumber_ -
+            firstSequenceNumberMap.get(streamInfo);
+        segmentIndex.dropFirstReferences(numSegmentsToDrop);
+
+        // Now adjust timestamps back to begin at 0.
+        const segmentN = segmentIndex.earliestReference();
+        if (segmentN) {
+          this.offsetStream_(streamInfo, -segmentN.startTime);
+        }
+      }
+    }
+
+    this.streamsSynced_ = true;
+  }
+
+  /**
+   * Synchronize streams by the EXT-X-PROGRAM-DATE-TIME tags attached to their
+   * segments.  Also normalizes segment times so that the earliest segment in
+   * any stream is at time 0.
+   * @private
+   */
+  syncStreamsWithProgramDateTime_() {
+    if (this.streamsSynced_) {
+      return;
+    }
+
+    let lowestSyncTime = Infinity;
+
+    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+      const segmentIndex = streamInfo.stream.segmentIndex;
+      if (segmentIndex) {
+        const segment0 = segmentIndex.earliestReference();
+        if (segment0 != null && segment0.syncTime != null) {
+          lowestSyncTime = Math.min(lowestSyncTime, segment0.syncTime);
+        }
       }
     }
-    if (segments.size > 0) {
-      this.syncTimeOffset_ = -lowestSyncTime;
-      for (const segment of segments) {
-        segment.syncTime += this.syncTimeOffset_;
-        for (const partial of segment.partialReferences) {
-          partial.syncTime += this.syncTimeOffset_;
+
+    if (lowestSyncTime == Infinity) {
+      // Nothing to sync.
+      return;
+    }
+
+    shaka.log.debug('Syncing HLS streams against base time:', lowestSyncTime);
+
+    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+      const segmentIndex = streamInfo.stream.segmentIndex;
+      if (segmentIndex != null) {
+        const segment0 = segmentIndex.earliestReference();
+        if (segment0.syncTime == null) {
+          shaka.log.alwaysError('Missing EXT-X-PROGRAM-DATE-TIME for stream',
+              streamInfo.verbatimMediaPlaylistUri,
+              'Expect AV sync issues!');
+        } else {
+          // The first segment's target startTime should be based entirely on
+          // its syncTime.  The rest of the stream will be based on that
+          // starting point.  The earliest segment sync time from any stream
+          // will become presentation time 0.  If two streams start e.g. 6
+          // seconds apart in syncTime, then their first segments will also
+          // start 6 seconds apart in presentation time.
+          const segment0TargetTime = segment0.syncTime - lowestSyncTime;
+          const streamOffset = segment0TargetTime - segment0.startTime;
+
+          this.offsetStream_(streamInfo, streamOffset);
         }
       }
     }
+
+    this.streamsSynced_ = true;
+  }
+
+  /**
+   * @param {!shaka.hls.HlsParser.StreamInfo} streamInfo
+   * @param {number} offset
+   * @private
+   */
+  offsetStream_(streamInfo, offset) {
+    streamInfo.stream.segmentIndex.offset(offset);
+
+    streamInfo.maxTimestamp += offset;
+    goog.asserts.assert(streamInfo.maxTimestamp >= 0,
+        'Negative maxTimestamp after adjustment!');
+
+    for (const [key, value] of streamInfo.mediaSequenceToStartTime) {
+      streamInfo.mediaSequenceToStartTime.set(key, value + offset);
+    }
+
+    shaka.log.debug('Offset', offset, 'applied to',
+        streamInfo.verbatimMediaPlaylistUri);
   }
 
   /**
@@ -525,7 +647,11 @@ shaka.hls.HlsParser = class {
 
     // Now that we have generated all streams, we can determine the offset to
     // apply to sync times.
-    this.calculateSyncTimeOffset_();
+    if (this.config_.hls.ignoreManifestProgramDateTime) {
+      this.syncStreamsWithSequenceNumber_();
+    } else {
+      this.syncStreamsWithProgramDateTime_();
+    }
 
     if (this.aesEncrypted_ && variants.length == 0) {
       // We do not support AES-128 encryption with HLS yet. Variants is null
@@ -540,15 +666,12 @@ shaka.hls.HlsParser = class {
 
     // Find the min and max timestamp of the earliest segment in all streams.
     // Find the minimum duration of all streams as well.
-    let minFirstTimestamp = Infinity;
     let minDuration = Infinity;
-
     for (const streamInfo of this.uriToStreamInfosMap_.values()) {
-      minFirstTimestamp =
-          Math.min(minFirstTimestamp, streamInfo.minTimestamp);
       if (streamInfo.stream.type != 'text') {
-        minDuration = Math.min(minDuration,
-            streamInfo.maxTimestamp - streamInfo.minTimestamp);
+        // Since everything is already offset to 0 (either by sync or by being
+        // VOD), only maxTimestamp is necessary to compute the duration.
+        minDuration = Math.min(minDuration, streamInfo.maxTimestamp);
       }
     }
 
@@ -592,18 +715,11 @@ shaka.hls.HlsParser = class {
             segmentAvailabilityDuration);
       }
     } else {
-      // For VOD/EVENT content, offset everything back to 0.
-      // Use the minimum timestamp as the offset for all streams.
       // Use the minimum duration as the presentation duration.
       this.presentationTimeline_.setDuration(minDuration);
-      // Use a negative offset to adjust towards 0.
-      this.presentationTimeline_.offset(-minFirstTimestamp);
 
       for (const streamInfo of this.uriToStreamInfosMap_.values()) {
-        // The segments were created with actual media times, rather than
-        // presentation-aligned times, so offset them all now.
-        streamInfo.stream.segmentIndex.offset(-minFirstTimestamp);
-        // Finally, fit the segments to the playlist duration.
+        // Fit the segments to the playlist duration.
         streamInfo.stream.segmentIndex.fit(/* periodStart= */ 0, minDuration);
       }
     }
@@ -811,12 +927,7 @@ shaka.hls.HlsParser = class {
     });
 
     // Create stream info for each audio / video media tag.
-    // Wait for the first stream info created, so that the start time is fetched
-    // and can be reused.
-    if (mediaTags.length) {
-      await this.createStreamInfoFromMediaTag_(mediaTags[0]);
-    }
-    const promises = mediaTags.slice(1).map((tag) => {
+    const promises = mediaTags.map((tag) => {
       return this.createStreamInfoFromMediaTag_(tag);
     });
     await Promise.all(promises);
@@ -1584,7 +1695,6 @@ shaka.hls.HlsParser = class {
       throw error;
     }
 
-    const minTimestamp = segments[0].startTime;
     const lastEndTime = segments[segments.length - 1].endTime;
     /** @type {!shaka.media.SegmentIndex} */
     const segmentIndex = new shaka.media.SegmentIndex(segments);
@@ -1642,7 +1752,6 @@ shaka.hls.HlsParser = class {
       stream,
       verbatimMediaPlaylistUri,
       absoluteMediaPlaylistUri,
-      minTimestamp,
       maxTimestamp: lastEndTime,
       mediaSequenceToStartTime,
       canSkipSegments,
@@ -1854,15 +1963,9 @@ shaka.hls.HlsParser = class {
       const dateTimeTag =
           shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-PROGRAM-DATE-TIME');
       if (dateTimeTag && dateTimeTag.value) {
-        const time = shaka.util.XmlUtils.parseDate(dateTimeTag.value);
-        goog.asserts.assert(time != null,
+        syncTime = shaka.util.XmlUtils.parseDate(dateTimeTag.value);
+        goog.asserts.assert(syncTime != null,
             'EXT-X-PROGRAM-DATE-TIME format not valid');
-        // Sync time offset is null on the first go-through. This indicates that
-        // we have not yet seen every stream, and thus do not yet have enough
-        // information to determine how to normalize the sync times.
-        // For that first go-through, the sync time will be applied after the
-        // references are all created. Until then, just offset by 0.
-        syncTime = time + (this.syncTimeOffset_ || 0);
       }
     }
 
@@ -2096,16 +2199,13 @@ shaka.hls.HlsParser = class {
       firstStartTime = mediaSequenceToStartTime.get(position);
     }
 
-    const firstSegmentUri = hlsSegments[0].absoluteUri;
-    shaka.log.debug('First segment', firstSegmentUri.split('/').pop(),
-        'starts at', firstStartTime);
-
     /** @type {!Array.<!shaka.media.SegmentReference>} */
     const references = [];
 
+    let previousReference = null;
+
     for (let i = 0; i < hlsSegments.length; i++) {
       const item = hlsSegments[i];
-      const previousReference = references[references.length - 1];
       const startTime =
           (i == 0) ? firstStartTime : previousReference.endTime;
       position = mediaSequenceNumber + skippedSegments + i;
@@ -2135,9 +2235,17 @@ shaka.hls.HlsParser = class {
           variables,
           playlist.absoluteUri,
           type);
+      previousReference = reference;
 
       if (reference) {
-        references.push(reference);
+        if (this.config_.hls.ignoreManifestProgramDateTime &&
+            this.minSequenceNumber_ != null &&
+            position < this.minSequenceNumber_) {
+          // This segment is ignored as part of our fallback synchronization
+          // method.
+        } else {
+          references.push(reference);
+        }
       }
     }
 
@@ -2593,7 +2701,6 @@ shaka.hls.HlsParser = class {
  *   stream: !shaka.extern.Stream,
  *   verbatimMediaPlaylistUri: string,
  *   absoluteMediaPlaylistUri: string,
- *   minTimestamp: number,
  *   maxTimestamp: number,
  *   mediaSequenceToStartTime: !Map.<number, number>,
  *   canSkipSegments: boolean
@@ -2612,8 +2719,6 @@ shaka.hls.HlsParser = class {
  * @property {string} absoluteMediaPlaylistUri
  *   The absolute media playlist URI, resolved relative to the master playlist
  *   and updated to reflect any redirects.
- * @property {number} minTimestamp
- *   The minimum timestamp found in the stream.
  * @property {number} maxTimestamp
  *   The maximum timestamp found in the stream.
  * @property {!Map.<number, number>} mediaSequenceToStartTime