feat: Add config to allow reset MSE on cross boundary (#8156)

There's devices out there that are not compliant with the MSE spec. Such as halting MSE when a secondary init segment is appended (webOS 3), or failing to transition from a plain to encrypted init segment (Tizen 2017). While we initially prefer content workarounds, it's a time consuming and trial & error process. For some devices it might not be worth investing time into finding a proper workaround due to low usage. We're giving people an alternative by resetting MSE when needed (configurable). dash.js offers somewhat similar behavior [here](https://github.com/Dash-Industry-Forum/dash.js/blob/a656ec709e7f92f76b392bf196ee9883da7928ce/src/streaming/controllers/StreamController.js#L672), where MSE is reset before applying an encrypted init segment. This PR introduces `crossBoundaryStrategy` in `StreamingConfiguration`. It can be configured as following: - KEEP - we're keeping MSE active, this is the default and the current behavior. - RESET - we'll always reset MSE when it crosses a boundary. - RESET_TO_ENCRYPTED - we reset MSE when it crosses an encrypted boundary, and we keep MSE afterwards. Additionally, we're not going to reset when we're crossing a plain to plain boundary. Each initSegmentReference now holds an `encrypted` and `boundaryEnd` value. When configured with a different value than KEEP, `StreamingEngine` will be instructed to fetch and append segment references up until the boundary of the currently applied init segment. We detect whether we're at a boundary in a few ways: - Listening to the HTML5 MediaElement's `waiting` event, this'll indicate that we do not have enough buffer to advance. If we're pretty close to the boundary, we assume we're at the boundary. - Due to subtle differences in the segment alignments, waiting wasn't reliable. When close to a boundary, a timer is fired with the assumption that "we'll reach the boundary at soon". I've set the threshold to 1 second, when playhead is further than the threshold, we'll skip checking whether an MSE reset is due. The implementation relies on the added properties in the init segment reference, and the concept of a "Period" is avoided in StreamingEngine to ensure it's compatible with HLS too. --------- Co-authored-by: Álvaro Velad Galván <[email protected]> Co-authored-by: Wojciech Tyczyński <[email protected]>
shaka-project · Mar 7, 2025 · 3699164 · 3699164
1 parent b8519f1
commit 3699164
Show file tree

Hide file tree

Showing 18 changed files with 408 additions and 7 deletions.
diff --git a/AUTHORS b/AUTHORS
@@ -35,6 +35,7 @@ Dave Nicholas <[email protected]>
 David Pfister <[email protected]>
 Davide Zordan <[email protected]>
 Dl Dador <[email protected]>
+DPG Media <*@dpgmedia.be>
 Edgeware AB <*@edgeware.tv>
 Enson Choy <[email protected]>
 Esteban Dosztal <[email protected]>

diff --git a/build/types/core b/build/types/core
@@ -7,6 +7,7 @@
 
 +../../lib/config/auto_show_text.js
 +../../lib/config/codec_switching_strategy.js
++../../lib/config/cross_boundary_strategy.js
 
 +../../lib/debug/asserts.js
 +../../lib/debug/log.js

diff --git a/demo/config.js b/demo/config.js
@@ -589,6 +589,13 @@ shakaDemo.Config = class {
     this.addSelectInput_('Preferred video layout', 'preferredVideoLayout',
         videoLayouts, videoLayoutsNames);
 
+    const strategyOptions = shaka.config.CrossBoundaryStrategy;
+    const strategyOptionsNames = {
+      'KEEP': 'Keep',
+      'RESET': 'Reset',
+      'RESET_TO_ENCRYPTED': 'Reset to encrypted',
+    };
+
     this.addBoolInput_('Start At Segment Boundary',
         'streaming.startAtSegmentBoundary')
         .addBoolInput_('Ignore Text Stream Failures',
@@ -608,7 +615,10 @@ shakaDemo.Config = class {
         .addBoolInput_('Should fix timestampOffset',
             'streaming.shouldFixTimestampOffset')
         .addBoolInput_('Avoid eviction on QuotaExceededError',
-            'streaming.avoidEvictionOnQuotaExceededError');
+            'streaming.avoidEvictionOnQuotaExceededError')
+        .addSelectInput_('Cross Boundary Strategy',
+            'streaming.crossBoundaryStrategy',
+            strategyOptions, strategyOptionsNames);
     this.addRetrySection_('streaming', 'Streaming Retry Parameters');
     this.addLiveSyncSection_();
   }

diff --git a/externs/shaka/player.js b/externs/shaka/player.js
@@ -1730,7 +1730,8 @@ shaka.extern.LiveSyncConfiguration;
  *   clearDecodingCache: boolean,
  *   dontChooseCodecs: boolean,
  *   shouldFixTimestampOffset: boolean,
- *   avoidEvictionOnQuotaExceededError: boolean
+ *   avoidEvictionOnQuotaExceededError: boolean,
+ *   crossBoundaryStrategy: shaka.config.CrossBoundaryStrategy
  * }}
  *
  * @description
@@ -1986,6 +1987,12 @@ shaka.extern.LiveSyncConfiguration;
  *   Avoid evict content on QuotaExceededError.
  *   <br>
  *   Defaults to <code>false</code>.
+ * @property {shaka.config.CrossBoundaryStrategy} crossBoundaryStrategy
+ *   Allows MSE to be reset when crossing a boundary. Optionally, we can stop
+ *   resetting MSE when MSE passed an encrypted boundary.
+ *   Defaults to <code>KEEP</code> except on Tizen 3 where the default value
+ *   is <code>RESET_TO_ENCRYPTED</code> and WebOS 3 where the default value
+ *   is <code>RESET</code>.
  * @exportDoc
  */
 shaka.extern.StreamingConfiguration;

diff --git a/lib/config/cross_boundary_strategy.js b/lib/config/cross_boundary_strategy.js
@@ -0,0 +1,27 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.config.CrossBoundaryStrategy');
+
+/**
+ * @enum {string}
+ * @export
+ */
+shaka.config.CrossBoundaryStrategy = {
+  /**
+   * Never reset MediaSource when crossing boundary.
+   */
+  'KEEP': 'keep',
+  /**
+   * Always reset MediaSource when crossing boundary.
+   */
+  'RESET': 'reset',
+  /**
+   * Reset MediaSource once, when transitioning from a plain
+   * boundary to an encrypted boundary.
+   */
+  'RESET_TO_ENCRYPTED': 'reset_to_encrypted',
+};
diff --git a/lib/dash/dash_parser.js b/lib/dash/dash_parser.js
@@ -2004,7 +2004,10 @@ shaka.dash.DashParser = class {
         this.config_.ignoreDrmInfo,
         this.config_.dash.keySystemsByURI);
 
-    context.adaptationSet.encrypted = contentProtection.drmInfos.length > 0;
+    // We us contentProtectionElements instead of drmInfos as the latter is
+    // not populated yet, and we need the encrypted flag for the upcoming
+    // parseRepresentation that will set the encrypted flag to the init seg.
+    context.adaptationSet.encrypted = contentProtectionElements.length > 0;
 
     const language = shaka.util.LanguageUtils.normalize(
         context.adaptationSet.language || 'und');
@@ -2216,6 +2219,8 @@ shaka.dash.DashParser = class {
         TXml.findChildren(node, 'SupplementalProperty');
     const essentialPropertyElements =
         TXml.findChildren(node, 'EssentialProperty');
+    const contentProtectionElements =
+        TXml.findChildren(node, 'ContentProtection');
 
     let representationUrlParams = null;
     let urlParamsElement = essentialPropertyElements.find((element) => {
@@ -2248,6 +2253,10 @@ shaka.dash.DashParser = class {
                    contentType == ContentType.APPLICATION;
     const isImage = contentType == ContentType.IMAGE;
 
+    if (contentProtectionElements.length) {
+      context.adaptationSet.encrypted = true;
+    }
+
     try {
       /** @type {shaka.extern.aesKey|undefined} */
       let aesKey = undefined;
@@ -2340,8 +2349,6 @@ shaka.dash.DashParser = class {
       throw error;
     }
 
-    const contentProtectionElements =
-        TXml.findChildren(node, 'ContentProtection');
     const keyId = shaka.dash.ContentProtection.parseFromRepresentation(
         contentProtectionElements, contentProtection,
         this.config_.ignoreDrmInfo,

diff --git a/lib/dash/segment_base.js b/lib/dash/segment_base.js
@@ -78,6 +78,9 @@ shaka.dash.SegmentBase = class {
         encrypted);
     ref.codecs = context.representation.codecs;
     ref.mimeType = context.representation.mimeType;
+    if (context.periodInfo) {
+      ref.boundaryEnd = context.periodInfo.start + context.periodInfo.duration;
+    }
     return ref;
   }
 

diff --git a/lib/dash/segment_template.js b/lib/dash/segment_template.js
@@ -702,6 +702,9 @@ shaka.dash.SegmentTemplate = class {
         encrypted);
     ref.codecs = context.representation.codecs;
     ref.mimeType = context.representation.mimeType;
+    if (context.periodInfo) {
+      ref.boundaryEnd = context.periodInfo.start + context.periodInfo.duration;
+    }
     return ref;
   }
 };

diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js
@@ -65,6 +65,9 @@ shaka.media.InitSegmentReference = class {
     /** @type {?string} */
     this.mimeType = null;
 
+    /** @type {?number} */
+    this.boundaryEnd = null;
+
     /** @const {boolean} */
     this.encrypted = encrypted;
   }

diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js
@@ -11,6 +11,7 @@
 goog.provide('shaka.media.StreamingEngine');
 
 goog.require('goog.asserts');
+goog.require('shaka.config.CrossBoundaryStrategy');
 goog.require('shaka.log');
 goog.require('shaka.media.Capabilities');
 goog.require('shaka.media.InitSegmentReference');
@@ -173,6 +174,22 @@ shaka.media.StreamingEngine = class {
         this.playerInterface_.mediaSourceEngine.clearLiveSeekableRange();
       }
     });
+
+    /** @private {?number} */
+    this.boundaryTime_ = null;
+
+    /** @private {?shaka.util.Timer} */
+    this.crossBoundaryTimer_ = new shaka.util.Timer(() => {
+      const video = this.playerInterface_.video;
+      if (video.ended) {
+        return;
+      }
+      if (this.boundaryTime_) {
+        shaka.log.info('Crossing boundary at', this.boundaryTime_);
+        video.currentTime = this.boundaryTime_;
+        this.boundaryTime_ = null;
+      }
+    });
   }
 
   /** @override */
@@ -211,6 +228,10 @@ shaka.media.StreamingEngine = class {
       this.updateLiveSeekableRangeTime_.stop();
     }
     this.updateLiveSeekableRangeTime_ = null;
+    if (this.crossBoundaryTimer_) {
+      this.crossBoundaryTimer_.stop();
+    }
+    this.crossBoundaryTimer_ = null;
   }
 
   /**
@@ -1522,6 +1543,14 @@ shaka.media.StreamingEngine = class {
       return updateIntervalSeconds;
     }
 
+    const CrossBoundaryStrategy = shaka.config.CrossBoundaryStrategy;
+    if (this.config_.crossBoundaryStrategy !== CrossBoundaryStrategy.KEEP &&
+        this.discardReferenceByBoundary_(mediaState, reference)) {
+      // Return null as we do not want to fetch and append segments outside
+      // of the current boundary.
+      return null;
+    }
+
     if (mediaState.segmentPrefetch && mediaState.segmentIterator &&
         !this.audioPrefetchMap_.has(mediaState.stream)) {
       mediaState.segmentPrefetch.evict(reference.startTime);
@@ -2979,6 +3008,109 @@ shaka.media.StreamingEngine = class {
     }
   }
 
+  /**
+   * Checks if need to push time forward to cross a boundary. If so,
+   * an MSE reset will happen. If the strategy is KEEP, this logic is skipped.
+   * Called on timeupdate to schedule a theoretical, future, offset or on
+   * waiting, which is another indicator we might need to cross a boundary.
+   * @param {boolean=} immediate
+   */
+  forwardTimeForCrossBoundary(immediate = false) {
+    if (this.config_.crossBoundaryStrategy ===
+          shaka.config.CrossBoundaryStrategy.KEEP) {
+      // When crossBoundaryStrategy changed to keep mid stream, we can bail
+      // out early.
+      return;
+    }
+    const video = this.playerInterface_.video;
+    if (video.seeking) {
+      // When seeking, close to a boundary, we can reset too early due to
+      // a subsequent waiting event. Schedule a theoretical delay.
+      immediate = false;
+    }
+
+    // Stop timer first, in case someone seeked back during the time a timer
+    // was scheduled.
+    this.crossBoundaryTimer_.stop();
+
+    const presentationTime = this.playerInterface_.getPresentationTime();
+
+    const ContentType = shaka.util.ManifestParserUtils.ContentType;
+    const mediaState = this.mediaStates_.get(ContentType.VIDEO) ||
+      this.mediaStates_.get(ContentType.AUDIO);
+
+    if (!mediaState || !mediaState.lastAppendWindowEnd ||
+        mediaState.clearingBuffer) {
+      return;
+    }
+
+    const threshold = shaka.media.StreamingEngine.CROSS_BOUNDARY_END_THRESHOLD_;
+    const fromEnd = mediaState.lastAppendWindowEnd - presentationTime;
+    // Check if greater than 0 to eliminate a backwards seek.
+    if (fromEnd > 0 && fromEnd < threshold) {
+      // Set the intended time to seek to in order to cross the boundary.
+      this.boundaryTime_ = mediaState.lastAppendWindowEnd;
+
+      if (immediate) {
+        this.crossBoundaryTimer_.tickNow();
+      } else {
+        // When not immediate, we schedule a time tick when the boundary
+        // theoretically should be reached, else we'd be stalled when
+        // a waiting event doesn't come (due to segment misalignment).
+        this.crossBoundaryTimer_.tickAfter(fromEnd);
+      }
+    }
+  }
+
+  /**
+   * Returns whether the reference should be discarded. If the segment crosses
+   * a boundary, we'll discard it based on the crossBoundaryStrategy.
+   *
+   * @param {!shaka.media.StreamingEngine.MediaState_} mediaState
+   * @param {!shaka.media.SegmentReference} reference
+   * @private
+   */
+  discardReferenceByBoundary_(mediaState, reference) {
+    const lastInitRef = mediaState.lastInitSegmentReference;
+    if (!lastInitRef) {
+      return false;
+    }
+
+    const CrossBoundaryStrategy = shaka.config.CrossBoundaryStrategy;
+    const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
+
+    const initRef = reference.initSegmentReference;
+    let discard = lastInitRef.boundaryEnd !== initRef.boundaryEnd;
+    // Some devices can play plain data when initialized with an encrypted
+    // init segment. We can keep the MediaSource in this case.
+    if (this.config_.crossBoundaryStrategy ===
+          CrossBoundaryStrategy.RESET_TO_ENCRYPTED) {
+      if (!lastInitRef.encrypted && !initRef.encrypted) {
+        // We're crossing a plain to plain boundary, allow the reference.
+        discard = false;
+      }
+      if (lastInitRef.encrypted) {
+        // We initialized MediaSource with an encrypted init segment, from
+        // now on, we can keep the buffer.
+        shaka.log.debug(logPrefix, 'stream is encrypted, ' +
+          'discard crossBoundaryStrategy');
+        this.config_.crossBoundaryStrategy = CrossBoundaryStrategy.KEEP;
+      }
+    }
+    // If discarded & seeked across a boundary, reset MediaSource.
+    if (discard && mediaState.seeked) {
+      shaka.log.debug(logPrefix, 'reset mediaSource',
+          'from=', mediaState.lastInitSegmentReference,
+          'to=', reference.initSegmentReference);
+
+      this.resetMediaSource(/* force= */ true).then(() => {
+        const eventName = shaka.util.FakeEvent.EventName.BoundaryCrossed;
+        this.playerInterface_.onEvent(new shaka.util.FakeEvent(eventName));
+      });
+    }
+    return discard;
+  }
+
   /**
    * @param {shaka.media.StreamingEngine.MediaState_} mediaState
    * @return {string} A log prefix of the form ($CONTENT_TYPE:$STREAM_ID), e.g.,
@@ -2996,6 +3128,7 @@ shaka.media.StreamingEngine = class {
  *   getPresentationTime: function():number,
  *   getBandwidthEstimate: function():number,
  *   getPlaybackRate: function():number,
+ *   video: !HTMLMediaElement,
  *   mediaSourceEngine: !shaka.media.MediaSourceEngine,
  *   netEngine: shaka.net.NetworkingEngine,
  *   onError: function(!shaka.util.Error),
@@ -3014,7 +3147,9 @@ shaka.media.StreamingEngine = class {
  * @property {function():number} getBandwidthEstimate
  *   Get the estimated bandwidth in bits per second.
  * @property {function():number} getPlaybackRate
- *   Get the playback rate
+ *   Get the playback rate.
+ * @property {!HTMLVideoElement} video
+ *   Get the video element.
  * @property {!shaka.media.MediaSourceEngine} mediaSourceEngine
  *   The MediaSourceEngine. The caller retains ownership.
  * @property {shaka.net.NetworkingEngine} netEngine
@@ -3176,3 +3311,13 @@ shaka.media.StreamingEngine.APPEND_WINDOW_END_FUDGE_ = 0.01;
  * @private
  */
 shaka.media.StreamingEngine.MAX_RUN_AHEAD_SEGMENTS_ = 1;
+
+
+/**
+ * The threshold to decide if we're close to a boundary. If presentation time
+ * is before this offset, boundary crossing logic will be skipped.
+ *
+ * @const {number}
+ * @private
+ */
+shaka.media.StreamingEngine.CROSS_BOUNDARY_END_THRESHOLD_ = 1;