diff --git a/lib/media/segment_index.js b/lib/media/segment_index.js
index ec6ed08d91..5c4b07eba6 100644
--- a/lib/media/segment_index.js
+++ b/lib/media/segment_index.js
@@ -700,6 +700,17 @@ shaka.media.MetaSegmentIndex = class extends shaka.media.SegmentIndex {
     this.indexes_ = [];
   }
 
+  /** Evicts all old SegmentIndexes in this MetaSegmentIndex that are empty. */
+  evictEmpty() {
+    while (this.indexes_.length > 1 && this.indexes_[0].isEmpty()) {
+      const index = this.indexes_.shift();
+      // In the case of this class, this.numEvicted_ represents the evicted
+      // segments that were in indexes that were entirely evicted.
+      this.numEvicted_ += index.getNumEvicted();
+      index.release();
+    }
+  }
+
   /**
    * Append a SegmentIndex to this MetaSegmentIndex.  This effectively stitches
    * the underlying Stream onto the end of the multi-Period Stream represented
@@ -724,6 +735,7 @@ shaka.media.MetaSegmentIndex = class extends shaka.media.SegmentIndex {
     // Be careful to clone the Array.  We don't want to share the reference with
     // our clone and affect each other accidentally.
     clone.indexes_ = this.indexes_.slice();
+    clone.numEvicted_ = this.numEvicted_;
     return clone;
   }
 
@@ -754,7 +766,7 @@ shaka.media.MetaSegmentIndex = class extends shaka.media.SegmentIndex {
    * @export
    */
   find(time) {
-    let numPassedInEarlierIndexes = 0;
+    let numPassedInEarlierIndexes = this.numEvicted_;
 
     for (const index of this.indexes_) {
       const position = index.find(time);
@@ -775,7 +787,7 @@ shaka.media.MetaSegmentIndex = class extends shaka.media.SegmentIndex {
    * @export
    */
   get(position) {
-    let numPassedInEarlierIndexes = 0;
+    let numPassedInEarlierIndexes = this.numEvicted_;
     let sawSegments = false;
 
     for (const index of this.indexes_) {
diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js
index 12d411f76d..9afb9c3f76 100644
--- a/lib/media/streaming_engine.js
+++ b/lib/media/streaming_engine.js
@@ -15,6 +15,7 @@ goog.require('shaka.log');
 goog.require('shaka.media.InitSegmentReference');
 goog.require('shaka.media.ManifestParser');
 goog.require('shaka.media.MediaSourceEngine');
+goog.require('shaka.media.MetaSegmentIndex');
 goog.require('shaka.media.SegmentIterator');
 goog.require('shaka.media.SegmentReference');
 goog.require('shaka.media.SegmentPrefetch');
@@ -2502,6 +2503,11 @@ shaka.media.StreamingEngine = class {
    * @private
    */
   async evict_(mediaState, presentationTime) {
+    const segmentIndex = mediaState.stream.segmentIndex;
+    if (segmentIndex instanceof shaka.media.MetaSegmentIndex) {
+      segmentIndex.evictEmpty();
+    }
+
     const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState);
     shaka.log.v2(logPrefix, 'checking buffer length');
 
diff --git a/test/media/segment_index_unit.js b/test/media/segment_index_unit.js
index 73e865b3d6..8cc8018cce 100644
--- a/test/media/segment_index_unit.js
+++ b/test/media/segment_index_unit.js
@@ -1040,6 +1040,27 @@ describe('SegmentIndex', /** @suppress {accessControls} */ () => {
       expect(Array.from(metaIndex)).toEqual(allRefs.slice(4));
     });
 
+    it('evicts empty SegmentIndexes when calling evictEmpty', () => {
+      const release0 = spyOn(index0, 'release');
+      const release1 = spyOn(index1, 'release');
+      const release2 = spyOn(index2, 'release');
+      metaIndex.appendSegmentIndex(index0);
+      metaIndex.appendSegmentIndex(index1);
+      metaIndex.appendSegmentIndex(index2);
+      expect(Array.from(metaIndex)).toEqual(
+          inputRefs0.concat(inputRefs1, inputRefs2));
+      expect(metaIndex.getNumEvicted()).toBe(0);
+      index0.evict(75);
+      index1.evict(75);
+      index2.evict(75);
+      metaIndex.evictEmpty();
+      expect(Array.from(metaIndex)).toEqual(inputRefs2.slice(1));
+      expect(release0).toHaveBeenCalled();
+      expect(release1).toHaveBeenCalled();
+      expect(release2).not.toHaveBeenCalled();
+      expect(metaIndex.getNumEvicted()).toBe(6);
+    });
+
     it('updates through updateEvery', async () => {
       metaIndex.appendSegmentIndex(index0);
       metaIndex.appendSegmentIndex(index1);