From c87c5bcdef3003d46efcd860dda6cf0030f1f4b4 Mon Sep 17 00:00:00 2001 From: Caitlin O'Callaghan <38890251+CaitlinOCallaghan@users.noreply.github.com> Date: Fri, 3 Sep 2021 09:57:43 -0700 Subject: [PATCH] Fix for gap size warning in Low Latency mode (#985) ## The issue - With LL-DASH mode enabled, the gap size warning was hit and printed to the console every time a new segment was registered to the manifest. - This occurred because the first chunk's size and duration were being stored for each segment, rather than the full segment size and duration. Note, only the first chunk's metrics are known at first because in low latency mode, the segment is registered to the manifest before it is finished being processed and written. - Because of this, the gap size check was comparing the end time of the first chunk in the previous segment to the beginning time of the current segment, causing the check to fail every time. ## The Fix - Update a low latency segment's duration and size once the segment file has been fully written. - The full segment size and duration will be used to update the bandwidth estimator and the segment info list. - Updating the segment info list to hold the full duration is necessary for satisfying [the gap size check found in Represenation.cc](https://github.com/google/shaka-packager/blob/master/packager/mpd/base/representation.cc#L391). - NOTE: bandwidth estimation is currently only used in HLS --- .../media/event/combined_muxer_listener.cc | 7 +++ .../media/event/combined_muxer_listener.h | 2 + .../media/event/mpd_notify_muxer_listener.cc | 6 ++ .../media/event/mpd_notify_muxer_listener.h | 2 + packager/media/event/muxer_listener.h | 17 +++++- .../mp4/low_latency_segment_segmenter.cc | 10 +++- .../mp4/low_latency_segment_segmenter.h | 1 + packager/mpd/base/mock_mpd_notifier.h | 2 + packager/mpd/base/mpd_notifier.h | 20 ++++++- packager/mpd/base/representation.cc | 30 ++++++++++ packager/mpd/base/representation.h | 22 +++++++- packager/mpd/base/representation_unittest.cc | 55 +++++++++++++++++++ packager/mpd/base/simple_mpd_notifier.cc | 13 +++++ packager/mpd/base/simple_mpd_notifier.h | 3 + 14 files changed, 181 insertions(+), 9 deletions(-) diff --git a/packager/media/event/combined_muxer_listener.cc b/packager/media/event/combined_muxer_listener.cc index 7e1bc445a48..a52bde47026 100644 --- a/packager/media/event/combined_muxer_listener.cc +++ b/packager/media/event/combined_muxer_listener.cc @@ -77,6 +77,13 @@ void CombinedMuxerListener::OnNewSegment(const std::string& file_name, } } +void CombinedMuxerListener::OnCompletedSegment(int64_t duration, + uint64_t segment_file_size) { + for (auto& listener : muxer_listeners_) { + listener->OnCompletedSegment(duration, segment_file_size); + } +} + void CombinedMuxerListener::OnKeyFrame(int64_t timestamp, uint64_t start_byte_offset, uint64_t size) { diff --git a/packager/media/event/combined_muxer_listener.h b/packager/media/event/combined_muxer_listener.h index ef66772bfdb..3d6fb91adb7 100644 --- a/packager/media/event/combined_muxer_listener.h +++ b/packager/media/event/combined_muxer_listener.h @@ -45,6 +45,8 @@ class CombinedMuxerListener : public MuxerListener { int64_t start_time, int64_t duration, uint64_t segment_file_size) override; + void OnCompletedSegment(int64_t duration, + uint64_t segment_file_size) override; void OnKeyFrame(int64_t timestamp, uint64_t start_byte_offset, uint64_t size); void OnCueEvent(int64_t timestamp, const std::string& cue_data) override; /// @} diff --git a/packager/media/event/mpd_notify_muxer_listener.cc b/packager/media/event/mpd_notify_muxer_listener.cc index d16d7b19667..4e96a5df0bf 100644 --- a/packager/media/event/mpd_notify_muxer_listener.cc +++ b/packager/media/event/mpd_notify_muxer_listener.cc @@ -204,6 +204,12 @@ void MpdNotifyMuxerListener::OnNewSegment(const std::string& file_name, } } +void MpdNotifyMuxerListener::OnCompletedSegment(int64_t duration, + uint64_t segment_file_size) { + mpd_notifier_->NotifyCompletedSegment(notification_id_.value(), duration, + segment_file_size); +} + void MpdNotifyMuxerListener::OnKeyFrame(int64_t timestamp, uint64_t start_byte_offset, uint64_t size) { diff --git a/packager/media/event/mpd_notify_muxer_listener.h b/packager/media/event/mpd_notify_muxer_listener.h index 271e7f3e255..8b62c4402dc 100644 --- a/packager/media/event/mpd_notify_muxer_listener.h +++ b/packager/media/event/mpd_notify_muxer_listener.h @@ -53,6 +53,8 @@ class MpdNotifyMuxerListener : public MuxerListener { int64_t start_time, int64_t duration, uint64_t segment_file_size) override; + void OnCompletedSegment(int64_t duration, + uint64_t segment_file_size) override; void OnKeyFrame(int64_t timestamp, uint64_t start_byte_offset, uint64_t size); void OnCueEvent(int64_t timestamp, const std::string& cue_data) override; /// @} diff --git a/packager/media/event/muxer_listener.h b/packager/media/event/muxer_listener.h index 1c1ee75a670..f5a9f4d3178 100644 --- a/packager/media/event/muxer_listener.h +++ b/packager/media/event/muxer_listener.h @@ -120,9 +120,11 @@ class MuxerListener { float duration_seconds) = 0; /// Called when a segment has been muxed and the file has been written. - /// Note: For some implementations, this is used to signal new subsegments. - /// For example, for generating video on demand (VOD) MPD manifest, this is - /// called to signal subsegments. + /// Note: For some implementations, this is used to signal new subsegments + /// or chunks. For example, for generating video on demand (VOD) MPD manifest, + /// this is called to signal subsegments. In the low latency case, this + /// indicates the start of a new segment and will contain info about the + /// segment's first chunk. /// @param segment_name is the name of the new segment. Note that some /// implementations may not require this, e.g. if this is a subsegment. /// @param start_time is the start time of the segment, relative to the @@ -135,6 +137,15 @@ class MuxerListener { int64_t duration, uint64_t segment_file_size) = 0; + /// Called when a segment has been muxed and the entire file has been written. + /// For Low Latency only. Note that it should be called after OnNewSegment. + /// When the low latency segment is initally added to the manifest, the size + /// and duration are not known, because the segment is still being processed. + /// This will update the segment's duration and size after the segment is + /// fully written and these values are known. + virtual void OnCompletedSegment(int64_t duration, + uint64_t segment_file_size) {} + /// Called when there is a new key frame. For Video only. Note that it should /// be called before OnNewSegment is called on the containing segment. /// @param timestamp is in terms of the timescale of the media. diff --git a/packager/media/formats/mp4/low_latency_segment_segmenter.cc b/packager/media/formats/mp4/low_latency_segment_segmenter.cc index 36574a633bb..fa8a8a4aebd 100644 --- a/packager/media/formats/mp4/low_latency_segment_segmenter.cc +++ b/packager/media/formats/mp4/low_latency_segment_segmenter.cc @@ -128,8 +128,8 @@ Status LowLatencySegmentSegmenter::WriteInitialChunk() { styp_->Write(buffer.get()); const size_t segment_header_size = buffer->Size(); - const size_t segment_size = segment_header_size + fragment_buffer()->Size(); - DCHECK_NE(segment_size, 0u); + segment_size_ = segment_header_size + fragment_buffer()->Size(); + DCHECK_NE(segment_size_, 0u); RETURN_IF_ERROR(buffer->WriteToFile(segment_file_.get())); if (muxer_listener()) { @@ -160,7 +160,7 @@ Status LowLatencySegmentSegmenter::WriteInitialChunk() { // Following chunks will be appended to the open segment file. muxer_listener()->OnNewSegment(file_name_, sidx()->earliest_presentation_time, - segment_duration, segment_size); + segment_duration, segment_size_); is_initial_chunk_in_seg_ = false; } @@ -179,6 +179,9 @@ Status LowLatencySegmentSegmenter::WriteChunk() { } Status LowLatencySegmentSegmenter::FinalizeSegment() { + if (muxer_listener()) { + muxer_listener()->OnCompletedSegment(GetSegmentDuration(), segment_size_); + } // Close the file now that the final chunk has been written if (!segment_file_.release()->Close()) { return Status( @@ -190,6 +193,7 @@ Status LowLatencySegmentSegmenter::FinalizeSegment() { // Current segment is complete. Reset state in preparation for the next // segment. is_initial_chunk_in_seg_ = true; + segment_size_ = 0u; num_segments_++; return Status::OK; diff --git a/packager/media/formats/mp4/low_latency_segment_segmenter.h b/packager/media/formats/mp4/low_latency_segment_segmenter.h index a3dc3cd1b75..9aa397c02af 100644 --- a/packager/media/formats/mp4/low_latency_segment_segmenter.h +++ b/packager/media/formats/mp4/low_latency_segment_segmenter.h @@ -61,6 +61,7 @@ class LowLatencySegmentSegmenter : public Segmenter { bool ll_dash_mpd_values_initialized_ = false; std::unique_ptr segment_file_; std::string file_name_; + size_t segment_size_ = 0u; DISALLOW_COPY_AND_ASSIGN(LowLatencySegmentSegmenter); }; diff --git a/packager/mpd/base/mock_mpd_notifier.h b/packager/mpd/base/mock_mpd_notifier.h index 60a4effdaaa..1c3d096df35 100644 --- a/packager/mpd/base/mock_mpd_notifier.h +++ b/packager/mpd/base/mock_mpd_notifier.h @@ -31,6 +31,8 @@ class MockMpdNotifier : public MpdNotifier { int64_t start_time, int64_t duration, uint64_t size)); + MOCK_METHOD3(NotifyCompletedSegment, + bool(uint32_t container_id, int64_t duration, uint64_t size)); MOCK_METHOD1(NotifyAvailabilityTimeOffset, bool(uint32_t container_id)); MOCK_METHOD1(NotifySegmentDuration, bool(uint32_t container_id)); MOCK_METHOD2(NotifyCueEvent, bool(uint32_t container_id, int64_t timestamp)); diff --git a/packager/mpd/base/mpd_notifier.h b/packager/mpd/base/mpd_notifier.h index f6e788004eb..349c84ea3aa 100644 --- a/packager/mpd/base/mpd_notifier.h +++ b/packager/mpd/base/mpd_notifier.h @@ -73,7 +73,8 @@ class MpdNotifier { virtual bool NotifySegmentDuration(uint32_t container_id) { return true; } /// Notifies MpdBuilder that there is a new segment ready. For live, this - /// is usually a new segment, for VOD this is usually a subsegment. + /// is usually a new segment, for VOD this is usually a subsegment, for low + /// latency this is the first chunk. /// @param container_id Container ID obtained from calling /// NotifyNewContainer(). /// @param start_time is the start time of the new segment, in units of the @@ -87,6 +88,23 @@ class MpdNotifier { int64_t duration, uint64_t size) = 0; + /// Notifies MpdBuilder that a segment is fully written and provides the + /// segment's complete duration and size. For Low Latency only. Note, size and + /// duration are not known when the low latency segment is first registered + /// with the MPD, so we must update these values after the segment is + /// complete. + /// @param container_id Container ID obtained from calling + /// NotifyNewContainer(). + /// @param duration is the duration of the complete segment, in units of the + /// stream's time scale. + /// @param size is the complete segment size in bytes. + /// @return true on success, false otherwise. + virtual bool NotifyCompletedSegment(uint32_t container_id, + int64_t duration, + uint64_t size) { + return true; + } + /// Notifies MpdBuilder that there is a new CueEvent. /// @param container_id Container ID obtained from calling /// NotifyNewContainer(). diff --git a/packager/mpd/base/representation.cc b/packager/mpd/base/representation.cc index 7aa5fb39c80..9a08c27f966 100644 --- a/packager/mpd/base/representation.cc +++ b/packager/mpd/base/representation.cc @@ -187,6 +187,29 @@ void Representation::AddNewSegment(int64_t start_time, state_change_listener_->OnNewSegmentForRepresentation(start_time, duration); AddSegmentInfo(start_time, duration); + + // Only update the buffer depth and bandwidth estimator when the full segment + // is completed. In the low latency case, only the first chunk in the segment + // has been written at this point. Therefore, we must wait until the entire + // segment has been written before updating buffer depth and bandwidth + // estimator. + if (!mpd_options_.mpd_params.low_latency_dash_mode) { + current_buffer_depth_ += segment_infos_.back().duration; + + bandwidth_estimator_.AddBlock(size, static_cast(duration) / + media_info_.reference_time_scale()); + } +} + +void Representation::UpdateCompletedSegment(int64_t duration, uint64_t size) { + if (!mpd_options_.mpd_params.low_latency_dash_mode) { + LOG(WARNING) + << "UpdateCompletedSegment is only applicable to low latency mode."; + return; + } + + UpdateSegmentInfo(duration); + current_buffer_depth_ += segment_infos_.back().duration; bandwidth_estimator_.AddBlock( @@ -410,6 +433,13 @@ void Representation::AddSegmentInfo(int64_t start_time, int64_t duration) { segment_infos_.push_back({start_time, adjusted_duration, kNoRepeat}); } +void Representation::UpdateSegmentInfo(int64_t duration) { + if (!segment_infos_.empty()) { + // Update the duration in the current segment. + segment_infos_.back().duration = duration; + } +} + bool Representation::ApproximiatelyEqual(int64_t time1, int64_t time2) const { if (!allow_approximate_segment_timeline_) return time1 == time2; diff --git a/packager/mpd/base/representation.h b/packager/mpd/base/representation.h index 97c9e1d4997..a1081b2b6f8 100644 --- a/packager/mpd/base/representation.h +++ b/packager/mpd/base/representation.h @@ -95,12 +95,25 @@ class Representation { /// @param start_time is the start time for the (sub)segment, in units of the /// stream's time scale. /// @param duration is the duration of the segment, in units of the stream's - /// time scale. - /// @param size of the segment in bytes. + /// time scale. In the low latency case, this duration is that of the + /// first chunk because the full duration is not yet known. + /// @param size of the segment in bytes. In the low latency case, this size is + /// that of the + /// first chunk because the full size is not yet known. virtual void AddNewSegment(int64_t start_time, int64_t duration, uint64_t size); + /// Update a media segment in the Representation. + /// In the low latency case, the segment duration will not be ready until the + /// entire segment has been processed. This allows setting the full duration + /// after the segment has been completed and the true duratio is known. + /// @param duration is the duration of the complete segment, in units of the + /// stream's + /// time scale. + /// @param size of the complete segment in bytes. + virtual void UpdateCompletedSegment(int64_t duration, uint64_t size); + /// Set the sample duration of this Representation. /// Sample duration is not available right away especially for live. This /// allows setting the sample duration after the Representation has been @@ -188,6 +201,11 @@ class Representation { // |allow_approximate_segment_timeline_| is set. void AddSegmentInfo(int64_t start_time, int64_t duration); + // Update the current SegmentInfo. This method is used to update the duration + // value after a low latency segment is complete, and the full segment + // duration is known. + void UpdateSegmentInfo(int64_t duration); + // Check if two timestamps are approximately equal if // |allow_approximate_segment_timeline_| is set; Otherwise check whether the // two times match. diff --git a/packager/mpd/base/representation_unittest.cc b/packager/mpd/base/representation_unittest.cc index 76cc73a5abf..18c28920f88 100644 --- a/packager/mpd/base/representation_unittest.cc +++ b/packager/mpd/base/representation_unittest.cc @@ -444,6 +444,8 @@ class SegmentTemplateTest : public RepresentationTest { public: void SetUp() override { mpd_options_.mpd_type = MpdType::kDynamic; + mpd_options_.mpd_params.low_latency_dash_mode = false; + representation_ = CreateRepresentation(ConvertToMediaInfo(GetDefaultMediaInfo()), kAnyRepresentationId, NoListener()); @@ -458,6 +460,16 @@ class SegmentTemplateTest : public RepresentationTest { SegmentInfo s = {start_time, duration, repeat}; segment_infos_for_expected_out_.push_back(s); + + if (mpd_options_.mpd_params.low_latency_dash_mode) { + // Low latency segments do not repeat, so create 1 new segment and return. + // At this point, only the first chunk of the low latency segment has been + // written. The bandwidth will be updated once the segment is fully + // written and the segment duration and size are known. + representation_->AddNewSegment(start_time, duration, size); + return; + } + if (repeat == 0) { expected_s_elements_ += base::StringPrintf(kSElementTemplateWithoutR, start_time, duration); @@ -474,6 +486,16 @@ class SegmentTemplateTest : public RepresentationTest { } } + void UpdateSegment(int64_t duration, uint64_t size) { + DCHECK(representation_); + DCHECK(!segment_infos_for_expected_out_.empty()); + + segment_infos_for_expected_out_.back().duration = duration; + representation_->UpdateCompletedSegment(duration, size); + bandwidth_estimator_.AddBlock( + size, static_cast(duration) / kDefaultTimeScale); + } + protected: std::string ExpectedXml() { const char kOutputTemplate[] = @@ -510,6 +532,39 @@ TEST_F(SegmentTemplateTest, OneSegmentNormal) { EXPECT_THAT(representation_->GetXml(), XmlNodeEqual(ExpectedXml())); } +TEST_F(SegmentTemplateTest, OneSegmentLowLatency) { + const int64_t kStartTime = 0; + const int64_t kChunkDuration = 5; + const uint64_t kChunkSize = 128; + const int64_t kSegmentDuration = kChunkDuration * 1000; + const uint64_t kSegmentSize = kChunkSize * 1000; + + mpd_options_.mpd_params.low_latency_dash_mode = true; + mpd_options_.mpd_params.target_segment_duration = + kSegmentDuration / representation_->GetMediaInfo().reference_time_scale(); + + // Set values used in LL-DASH MPD attributes + representation_->SetSampleDuration(kChunkDuration); + representation_->SetAvailabilityTimeOffset(); + representation_->SetSegmentDuration(); + + // Register segment after the first chunk is complete + AddSegments(kStartTime, kChunkDuration, kChunkSize, 0); + // Update SegmentInfo after the segment is complete + UpdateSegment(kSegmentDuration, kSegmentSize); + + const char kOutputTemplate[] = + "\n" + " \n" + "\n"; + EXPECT_THAT(representation_->GetXml(), XmlNodeEqual(kOutputTemplate)); +} + TEST_F(SegmentTemplateTest, RepresentationClone) { MediaInfo media_info = ConvertToMediaInfo(GetDefaultMediaInfo()); media_info.set_segment_template_url("$Number$.mp4"); diff --git a/packager/mpd/base/simple_mpd_notifier.cc b/packager/mpd/base/simple_mpd_notifier.cc index 1dbc76f5738..76f03842701 100644 --- a/packager/mpd/base/simple_mpd_notifier.cc +++ b/packager/mpd/base/simple_mpd_notifier.cc @@ -119,6 +119,19 @@ bool SimpleMpdNotifier::NotifyNewSegment(uint32_t container_id, return true; } +bool SimpleMpdNotifier::NotifyCompletedSegment(uint32_t container_id, + int64_t duration, + uint64_t size) { + base::AutoLock auto_lock(lock_); + auto it = representation_map_.find(container_id); + if (it == representation_map_.end()) { + LOG(ERROR) << "Unexpected container_id: " << container_id; + return false; + } + it->second->UpdateCompletedSegment(duration, size); + return true; +} + bool SimpleMpdNotifier::NotifyCueEvent(uint32_t container_id, int64_t timestamp) { base::AutoLock auto_lock(lock_); diff --git a/packager/mpd/base/simple_mpd_notifier.h b/packager/mpd/base/simple_mpd_notifier.h index 783605c4336..9da46a221af 100644 --- a/packager/mpd/base/simple_mpd_notifier.h +++ b/packager/mpd/base/simple_mpd_notifier.h @@ -44,6 +44,9 @@ class SimpleMpdNotifier : public MpdNotifier { int64_t start_time, int64_t duration, uint64_t size) override; + bool NotifyCompletedSegment(uint32_t container_id, + int64_t duration, + uint64_t size) override; bool NotifyCueEvent(uint32_t container_id, int64_t timestamp) override; bool NotifyEncryptionUpdate(uint32_t container_id, const std::string& drm_uuid,