diff --git a/lib/media/playhead_observer.js b/lib/media/playhead_observer.js
index a3da10bd59..341e243021 100644
--- a/lib/media/playhead_observer.js
+++ b/lib/media/playhead_observer.js
@@ -7,6 +7,7 @@
 goog.provide('shaka.media.IPlayheadObserver');
 goog.provide('shaka.media.PlayheadObserverManager');
 
+goog.require('shaka.util.EventManager');
 goog.require('shaka.util.IReleasable');
 goog.require('shaka.util.Timer');
 
@@ -47,6 +48,9 @@ shaka.media.PlayheadObserverManager = class {
     /** @private {HTMLMediaElement} */
     this.mediaElement_ = mediaElement;
 
+    /** @private {shaka.util.EventManager} */
+    this.eventManager_ = new shaka.util.EventManager();
+
     /**
      * The set of all observers that this manager is responsible for updating.
      * We are using a set to ensure that we don't double update an observer if
@@ -65,11 +69,27 @@ shaka.media.PlayheadObserverManager = class {
      */
     this.pollingLoop_ = new shaka.util.Timer(() => {
       this.pollAllObservers_(/* seeking= */ false);
-    }).tickEvery(/* seconds= */ 0.25);
+    }).tickNow();
+
+    if (!mediaElement.paused) {
+      this.pollingLoop_.tickEvery(/* seconds= */ 0.25);
+    }
+
+    this.eventManager_.listen(mediaElement, 'playing', () => {
+      this.pollingLoop_.tickNow().tickEvery(/* seconds= */ 0.25);
+    });
+    this.eventManager_.listen(mediaElement, 'pause', () => {
+      this.pollingLoop_.stop();
+    });
   }
 
   /** @override */
   release() {
+    if (this.eventManager_) {
+      this.eventManager_.release();
+      this.eventManager_ = null;
+    }
+
     // We need to stop the loop or else we may try to use a released resource.
     this.pollingLoop_.stop();
 
@@ -103,10 +123,9 @@ shaka.media.PlayheadObserverManager = class {
    * @private
    */
   pollAllObservers_(seeking) {
+    const currentTime = this.mediaElement_.currentTime;
     for (const observer of this.observers_) {
-      observer.poll(
-          this.mediaElement_.currentTime,
-          seeking);
+      observer.poll(currentTime, seeking);
     }
   }
 };