diff --git a/api/video/builtin_video_bitrate_allocator_factory.cc b/api/video/builtin_video_bitrate_allocator_factory.cc index 4c24a0e75d..567fd5c53f 100644 --- a/api/video/builtin_video_bitrate_allocator_factory.cc +++ b/api/video/builtin_video_bitrate_allocator_factory.cc @@ -34,6 +34,7 @@ class BuiltinVideoBitrateAllocatorFactory case kVideoCodecAV1: case kVideoCodecVP9: return std::make_unique(codec); + // TODO: add an allocator here for H.265 default: return std::make_unique(codec); } diff --git a/api/video_codecs/video_codec.cc b/api/video_codecs/video_codec.cc index 7a500f9b0b..e93ec7c18a 100644 --- a/api/video_codecs/video_codec.cc +++ b/api/video_codecs/video_codec.cc @@ -26,9 +26,7 @@ constexpr char kPayloadNameAv1[] = "AV1"; // needed. constexpr char kPayloadNameAv1x[] = "AV1X"; constexpr char kPayloadNameH264[] = "H264"; -#ifdef WEBRTC_USE_H265 constexpr char kPayloadNameH265[] = "H265"; -#endif constexpr char kPayloadNameGeneric[] = "Generic"; constexpr char kPayloadNameMultiplex[] = "Multiplex"; } // namespace @@ -55,6 +53,17 @@ bool VideoCodecH264::operator==(const VideoCodecH264& other) const { numberOfTemporalLayers == other.numberOfTemporalLayers); } +#ifndef DISABLE_H265 +bool VideoCodecH265::operator==(const VideoCodecH265& other) const { + return (frameDroppingOn == other.frameDroppingOn && + keyFrameInterval == other.keyFrameInterval && + vpsLen == other.vpsLen && spsLen == other.spsLen && + ppsLen == other.ppsLen && + (spsLen == 0 || memcmp(spsData, other.spsData, spsLen) == 0) && + (ppsLen == 0 || memcmp(ppsData, other.ppsData, ppsLen) == 0)); +} +#endif + #ifdef WEBRTC_USE_H265 bool VideoCodecH265::operator==(const VideoCodecH265& other) const { return (frameDroppingOn == other.frameDroppingOn && @@ -116,7 +125,6 @@ const VideoCodecH264& VideoCodec::H264() const { return codec_specific_.H264; } -#ifdef WEBRTC_USE_H265 VideoCodecH265* VideoCodec::H265() { RTC_DCHECK_EQ(codecType, kVideoCodecH265); return &codec_specific_.H265; @@ -138,7 +146,6 @@ const char* CodecTypeToPayloadString(VideoCodecType type) { return kPayloadNameAv1; case kVideoCodecH264: return kPayloadNameH264; -#ifdef WEBRTC_USE_H265 case kVideoCodecH265: return kPayloadNameH265; #endif @@ -163,7 +170,6 @@ VideoCodecType PayloadStringToCodecType(const std::string& name) { return kVideoCodecH264; if (absl::EqualsIgnoreCase(name, kPayloadNameMultiplex)) return kVideoCodecMultiplex; -#ifdef WEBRTC_USE_H265 if (absl::EqualsIgnoreCase(name, kPayloadNameH265)) return kVideoCodecH265; #endif diff --git a/api/video_codecs/video_codec.h b/api/video_codecs/video_codec.h index 074e0fecdb..080425a310 100644 --- a/api/video_codecs/video_codec.h +++ b/api/video_codecs/video_codec.h @@ -112,6 +112,23 @@ struct VideoCodecH265 { size_t ppsLen; }; +#ifndef DISABLE_H265 +struct VideoCodecH265 { + bool operator==(const VideoCodecH265& other) const; + bool operator!=(const VideoCodecH265& other) const { + return !(*this == other); + } + bool frameDroppingOn; + int keyFrameInterval; + const uint8_t* vpsData; + size_t vpsLen; + const uint8_t* spsData; + size_t spsLen; + const uint8_t* ppsData; + size_t ppsLen; +}; +#endif + // Translates from name of codec to codec type and vice versa. RTC_EXPORT const char* CodecTypeToPayloadString(VideoCodecType type); RTC_EXPORT VideoCodecType PayloadStringToCodecType(const std::string& name); @@ -120,7 +137,9 @@ union VideoCodecUnion { VideoCodecVP8 VP8; VideoCodecVP9 VP9; VideoCodecH264 H264; +#ifndef DISABLE_H265 VideoCodecH265 H265; +#endif }; enum class VideoCodecMode { kRealtimeVideo, kScreensharing }; diff --git a/build_overrides/build.gni b/build_overrides/build.gni index 137b6a40b2..ba5dbaf4a4 100644 --- a/build_overrides/build.gni +++ b/build_overrides/build.gni @@ -32,6 +32,12 @@ ubsan_vptr_ignorelist_path = # so we just ignore that assert. See https://crbug.com/648948 for more info. ignore_elf32_limitations = true +if (is_win || is_ios || is_android) { + rtc_use_h265 = true +} else { + rtc_use_h265 = false +} + # Use bundled hermetic Xcode installation maintainted by Chromium, # except for local iOS builds where it's unsupported. # Allow for mac cross compile on linux machines. diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index a0463b357a..6d430bda9d 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -349,7 +349,9 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, is_keyframe, rtp_video_header); } return; +#ifndef DISABLE_H265 case VideoCodecType::kVideoCodecH265: +#endif case VideoCodecType::kVideoCodecMultiplex: return; } diff --git a/media/base/media_constants.cc b/media/base/media_constants.cc index cf72fc9993..689af3bacd 100644 --- a/media/base/media_constants.cc +++ b/media/base/media_constants.cc @@ -116,7 +116,6 @@ const char kH264FmtpSpropParameterSets[] = "sprop-parameter-sets"; const char kH264FmtpSpsPpsIdrInKeyframe[] = "sps-pps-idr-in-keyframe"; const char kH264ProfileLevelConstrainedBaseline[] = "42e01f"; const char kH264ProfileLevelConstrainedHigh[] = "640c1f"; -#ifdef WEBRTC_USE_H265 // RFC 7798 RTP Payload Format for H.265 video const char kH265FmtpProfileSpace[] = "profile-space"; const char kH265FmtpProfileId[] = "profile-id"; diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 72162043f7..da1afdd08b 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -255,6 +255,19 @@ rtc_library("rtp_rtcp") { defines = [ "BWE_TEST_LOGGING_COMPILE_TIME_ENABLE=0" ] } + if (rtc_use_h265) { + sources += [ + "source/rtp_format_h265.cc", + "source/rtp_format_h265.h", + "source/video_rtp_depacketizer_h265.cc", + "source/video_rtp_depacketizer_h265.h", + ] + } + + if (!rtc_use_h265) { + defines += ["DISABLE_H265"] + } + deps = [ ":rtp_rtcp_format", ":rtp_video_header", diff --git a/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc b/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc index aaa0ba3f57..4c6e525ed8 100644 --- a/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc +++ b/modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc @@ -19,8 +19,9 @@ #include "modules/rtp_rtcp/source/video_rtp_depacketizer_h264.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp8.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h" +#ifndef DISABLE_H265 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_h265.h" - +#endif namespace webrtc { std::unique_ptr CreateVideoRtpDepacketizer( @@ -32,8 +33,10 @@ std::unique_ptr CreateVideoRtpDepacketizer( return std::make_unique(); case kVideoCodecVP9: return std::make_unique(); +#ifndef DISABLE_H265 case kVideoCodecH265: return std::make_unique(); +#endif case kVideoCodecAV1: return std::make_unique(); case kVideoCodecGeneric: diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 3ff884034f..70b0fe5517 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -815,7 +815,9 @@ uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) { uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) { return kNoTemporalIdx; } +#ifndef DISABLE_H265 uint8_t operator()(const RTPVideoHeaderH265&) { return kNoTemporalIdx; } +#endif uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; } }; return absl::visit(TemporalIdGetter(), header.video_type_header); diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index e10de135b7..8837bc5b86 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -37,12 +37,20 @@ struct RTPVideoHeaderLegacyGeneric { uint16_t picture_id; }; +#ifndef DISABLE_H265 using RTPVideoTypeHeader = absl::variant; + RTPVideoHeaderLegacyGeneric>; +#else +using RTPVideoTypeHeader = absl::variant; +#endif struct RTPVideoHeader { struct GenericDescriptorInfo { diff --git a/modules/video_coding/encoded_frame.cc b/modules/video_coding/encoded_frame.cc index 7fba866497..b72cb98d73 100644 --- a/modules/video_coding/encoded_frame.cc +++ b/modules/video_coding/encoded_frame.cc @@ -145,6 +145,11 @@ void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) { _codecSpecificInfo.codecType = kVideoCodecH265; break; } +#ifndef DISABLE_H265 + case kVideoCodecH265: { + _codecSpecificInfo.codecType = kVideoCodecH265; + break; + } #endif default: { _codecSpecificInfo.codecType = kVideoCodecGeneric; diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index be2a31fcf6..6f965c8382 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -20,7 +20,6 @@ #include "api/video_codecs/video_encoder.h" #include "common_video/generic_frame_descriptor/generic_frame_info.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" -#ifdef WEBRTC_USE_H265 #include "modules/video_coding/codecs/h265/include/h265_globals.h" #endif #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -94,7 +93,6 @@ struct CodecSpecificInfoH264 { bool idr_frame; }; -#ifdef WEBRTC_USE_H265 struct CodecSpecificInfoH265 { H265PacketizationMode packetization_mode; bool idr_frame; diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index f3907a1293..909ad26bd5 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -267,6 +267,7 @@ std::vector> PacketBuffer::FindFrames( bool has_h264_pps = false; bool has_h264_idr = false; bool is_h264_keyframe = false; + bool is_h265 = false; #ifdef WEBRTC_USE_H265 is_h265 = buffer_[start_index]->codec() == kVideoCodecH265; @@ -442,7 +443,6 @@ std::vector> PacketBuffer::FindFrames( } } #endif - if (is_h264 || full_frame_found) { const uint16_t end_seq_num = seq_num + 1; // Use uint16_t type to handle sequence number wrap around case. uint16_t num_packets = end_seq_num - start_seq_num; diff --git a/rtc_base/experiments/min_video_bitrate_experiment.cc b/rtc_base/experiments/min_video_bitrate_experiment.cc index bb43e6df15..8879782837 100644 --- a/rtc_base/experiments/min_video_bitrate_experiment.cc +++ b/rtc_base/experiments/min_video_bitrate_experiment.cc @@ -100,7 +100,9 @@ absl::optional GetExperimentalMinVideoBitrate(VideoCodecType type) { return min_bitrate_av1.GetOptional(); case kVideoCodecH264: return min_bitrate_h264.GetOptional(); +#ifndef DISABLE_H265 case kVideoCodecH265: +#endif case kVideoCodecGeneric: case kVideoCodecMultiplex: return absl::nullopt; diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index db5c00558e..a0ddaa6307 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -1312,6 +1312,11 @@ if (is_ios || is_mac) { "objc/components/video_codec/RTCVideoDecoderH264.h", "objc/components/video_codec/RTCVideoEncoderFactoryH264.h", "objc/components/video_codec/RTCVideoEncoderH264.h", + "objc/components/video_codec/RTCH265ProfileLevelId.h", + "objc/components/video_codec/RTCVideoDecoderFactoryH265.h", + "objc/components/video_codec/RTCVideoDecoderH265.h", + "objc/components/video_codec/RTCVideoEncoderFactoryH265.h", + "objc/components/video_codec/RTCVideoEncoderH265.h", "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", "objc/helpers/RTCCameraPreviewView.h", "objc/helpers/RTCDispatcher.h", @@ -1401,6 +1406,7 @@ if (is_ios || is_mac) { ":videocodec_objc", ":videotoolbox_objc", "../api:create_peerconnection_factory", + "../api:create_peerconnection_factory", ] if (rtc_ios_macos_use_opengl_rendering) { deps += [ ":opengl_ui_objc" ] @@ -1516,6 +1522,11 @@ if (is_ios || is_mac) { "objc/components/video_codec/RTCVideoDecoderH264.h", "objc/components/video_codec/RTCVideoEncoderFactoryH264.h", "objc/components/video_codec/RTCVideoEncoderH264.h", + "objc/components/video_codec/RTCH265ProfileLevelId.h", + "objc/components/video_codec/RTCVideoDecoderFactoryH265.h", + "objc/components/video_codec/RTCVideoDecoderH265.h", + "objc/components/video_codec/RTCVideoEncoderFactoryH265.h", + "objc/components/video_codec/RTCVideoEncoderH265.h", "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", "objc/helpers/RTCDispatcher.h", ] @@ -1744,6 +1755,14 @@ if (is_ios || is_mac) { "objc/components/video_codec/RTCVideoEncoderFactoryH264.m", "objc/components/video_codec/RTCVideoEncoderH264.h", "objc/components/video_codec/RTCVideoEncoderH264.mm", + "objc/components/video_codec/RTCVideoDecoderFactoryH265.h", + "objc/components/video_codec/RTCVideoDecoderFactoryH265.m", + "objc/components/video_codec/RTCVideoDecoderH265.h", + "objc/components/video_codec/RTCVideoDecoderH265.mm", + "objc/components/video_codec/RTCVideoEncoderFactoryH265.h", + "objc/components/video_codec/RTCVideoEncoderFactoryH265.m", + "objc/components/video_codec/RTCVideoEncoderH265.h", + "objc/components/video_codec/RTCVideoEncoderH265.mm", ] if (rtc_use_h265) { diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java index 200e7a7698..7cab2de24d 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java @@ -137,7 +137,7 @@ public VideoEncoder createEncoder(VideoCodecInfo input) { public VideoCodecInfo[] getSupportedCodecs() { List supportedCodecInfos = new ArrayList(); // Generate a list of supported codecs in order of preference: - // VP8, VP9, H264 (high profile), and H264 (baseline profile). + // VP8, VP9, H.265(optional), H264 (high profile), and H264 (baseline profile). for (VideoCodecMimeType type : new VideoCodecMimeType[] {VideoCodecMimeType.VP8, VideoCodecMimeType.VP8, VideoCodecMimeType.VP9, VideoCodecMimeType.H264, VideoCodecMimeType.H265}) { @@ -257,6 +257,15 @@ private boolean isHardwareSupportedInCurrentSdkH265(MediaCodecInfo info) { || vcp.isExtraHardwareSupported(name, "video/hevc", vcp.parseWithTag(vcp.loadWithDom(extraMediaCodecFile), "Decoders")); } + private boolean isHardwareSupportedInCurrentSdkH265(MediaCodecInfo info) { + String name = info.getName(); + // QCOM H265 encoder is supported in KITKAT or later. + return (name.startsWith(QCOM_PREFIX) && Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) + // Exynos H265 encoder is supported in LOLLIPOP or later. + || (name.startsWith(EXYNOS_PREFIX) + && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP); + } + private boolean isMediaCodecAllowed(MediaCodecInfo info) { if (codecAllowedPredicate == null) { return true; diff --git a/sdk/android/src/jni/android_media_codec_common.h b/sdk/android/src/jni/android_media_codec_common.h new file mode 100644 index 0000000000..1f024ef78e --- /dev/null +++ b/sdk/android/src/jni/android_media_codec_common.h @@ -0,0 +1,90 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_ANDROID_SRC_JNI_ANDROID_MEDIA_CODEC_COMMON_H_ +#define SDK_ANDROID_SRC_JNI_ANDROID_MEDIA_CODEC_COMMON_H_ + +#include + +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +// Uncomment this define to enable verbose logging for every encoded/decoded +// video frame. +//#define TRACK_BUFFER_TIMING + +#define TAG_COMMON "MediaCodecVideo" + +// Color formats supported by encoder or decoder - should include all +// colors from supportedColorList in MediaCodecVideoEncoder.java and +// MediaCodecVideoDecoder.java. Supported color format set in encoder +// and decoder could be different. +enum COLOR_FORMATTYPE { + COLOR_FormatYUV420Planar = 0x13, + COLOR_FormatYUV420SemiPlanar = 0x15, + COLOR_QCOM_FormatYUV420SemiPlanar = 0x7FA30C00, + // NV12 color format supported by QCOM codec, but not declared in MediaCodec - + // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h + // This format is presumably similar to COLOR_FormatYUV420SemiPlanar, + // but requires some (16, 32?) byte alignment. + COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka = 0x7FA30C01, + COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka = 0x7FA30C02, + COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka = 0x7FA30C03, + COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04 +}; + +// Arbitrary interval to poll the codec for new outputs. +enum { kMediaCodecPollMs = 10 }; +// Arbitrary interval to poll at when there should be no more frames. +enum { kMediaCodecPollNoFramesMs = 100 }; +// Media codec maximum output buffer ready timeout. +enum { kMediaCodecTimeoutMs = 1000 }; +// Interval to print codec statistics (bitrate, fps, encoding/decoding time). +enum { kMediaCodecStatisticsIntervalMs = 3000 }; +// Maximum amount of pending frames for VP8 decoder. +enum { kMaxPendingFramesVp8 = 1 }; +// Maximum amount of pending frames for VP9 decoder. +enum { kMaxPendingFramesVp9 = 1 }; +// Maximum amount of pending frames for H.264 decoder. +enum { kMaxPendingFramesH264 = 4 }; +// TODO(zhanghe): update the value after checking more devices if needed. +// Maximum amount of pending frames for H.265 decoder. +enum { kMaxPendingFramesH265 = 4 }; +// Maximum amount of decoded frames for which per-frame logging is enabled. +enum { kMaxDecodedLogFrames = 10 }; +// Maximum amount of encoded frames for which per-frame logging is enabled. +enum { kMaxEncodedLogFrames = 10 }; + +static inline void AllowBlockingCalls() { + rtc::Thread* current_thread = rtc::Thread::Current(); + if (current_thread != NULL) + current_thread->DEPRECATED_AllowBlockingCalls(); +} + +// Checks for any Java exception, prints stack backtrace and clears +// currently thrown exception. +static inline bool CheckException(JNIEnv* jni) { + if (jni->ExceptionCheck()) { + RTC_LOG_TAG(rtc::LS_ERROR, TAG_COMMON) << "Java JNI exception."; + jni->ExceptionDescribe(); + jni->ExceptionClear(); + return true; + } + return false; +} + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_ANDROID_MEDIA_CODEC_COMMON_H_ diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h index 03caa2d869..ea2649090e 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h @@ -73,6 +73,7 @@ NS_ASSUME_NONNULL_BEGIN decoderFactory:(nullable id)decoderFactory; + /** Initialize an RTCPeerConnection with a configuration, constraints, and * dependencies. */ @@ -86,7 +87,10 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)initWithNativePeerConnectionFactory:( rtc::scoped_refptr)factory; -@end + +- (instancetype)initWithNativePeerConnectionFactory:( + rtc::scoped_refptr)factory; + NS_ASSUME_NONNULL_END diff --git a/test/scenario/video_stream.cc b/test/scenario/video_stream.cc index b54459e956..d16b3a80f8 100644 --- a/test/scenario/video_stream.cc +++ b/test/scenario/video_stream.cc @@ -204,6 +204,20 @@ CreateH264SpecificSettings(VideoStreamConfig config) { return nullptr; } +#ifdef WEBRTC_USE_H265 +rtc::scoped_refptr +CreateH265SpecificSettings(VideoStreamConfig config) { + RTC_DCHECK_EQ(config.encoder.layers.temporal, 1); + RTC_DCHECK_EQ(config.encoder.layers.spatial, 1); + + VideoCodecH265 h265_settings = VideoEncoder::GetDefaultH265Settings(); + h265_settings.frameDroppingOn = config.encoder.frame_dropping; + h265_settings.keyFrameInterval = + config.encoder.key_frame_interval.value_or(0); + return new rtc::RefCountedObject< + VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings); +} + #ifdef WEBRTC_USE_H265 rtc::scoped_refptr CreateH265SpecificSettings(VideoStreamConfig config) { diff --git a/video/config/video_encoder_config.cc b/video/config/video_encoder_config.cc index add489ef03..cc4d67e562 100644 --- a/video/config/video_encoder_config.cc +++ b/video/config/video_encoder_config.cc @@ -126,6 +126,17 @@ void VideoEncoderConfig::H265EncoderSpecificSettings::FillVideoCodecH265( *h265_settings = specifics_; } +#ifndef DISABLE_H265 +VideoEncoderConfig::H265EncoderSpecificSettings::H265EncoderSpecificSettings( + const VideoCodecH265& specifics) + : specifics_(specifics) {} + +void VideoEncoderConfig::H265EncoderSpecificSettings::FillVideoCodecH265( + VideoCodecH265* h265_settings) const { + *h265_settings = specifics_; +} +#endif + VideoEncoderConfig::Vp8EncoderSpecificSettings::Vp8EncoderSpecificSettings( const VideoCodecVP8& specifics) : specifics_(specifics) {} diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h new file mode 100644 index 0000000000..f4dc06dbec --- /dev/null +++ b/video/rtp_video_stream_receiver.h @@ -0,0 +1,373 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ +#define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/video/color_space.h" +#include "api/video_codecs/video_codec.h" +#include "call/rtp_packet_sink_interface.h" +#include "call/syncable.h" +#include "call/video_receive_stream.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/include/rtp_rtcp.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/loss_notification_controller.h" +#ifndef DISABLE_H265 +#include "modules/video_coding/h265_vps_sps_pps_tracker.h" +#endif +#include "modules/video_coding/packet_buffer.h" +#include "modules/video_coding/rtp_frame_reference_finder.h" +#include "modules/video_coding/unique_timestamp_counter.h" +#include "rtc_base/constructor_magic.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" +#include "video/buffered_frame_decryptor.h" +#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" + +namespace webrtc { + +class NackModule; +class PacketRouter; +class ProcessThread; +class ReceiveStatistics; +class ReceiveStatisticsProxy; +class RtcpRttStats; +class RtpPacketReceived; +class Transport; +class UlpfecReceiver; + +class RtpVideoStreamReceiver : public LossNotificationSender, + public RecoveredPacketReceiver, + public RtpPacketSinkInterface, + public KeyFrameRequestSender, + public video_coding::OnCompleteFrameCallback, + public OnDecryptedFrameCallback, + public OnDecryptionStatusChangeCallback { + public: + RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer); + ~RtpVideoStreamReceiver() override; + + void AddReceiveCodec(const VideoCodec& video_codec, + const std::map& codec_params, + bool raw_payload); + + void StartReceive(); + void StopReceive(); + + // Produces the transport-related timestamps; current_delay_ms is left unset. + absl::optional GetSyncInfo() const; + + bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); + + void FrameContinuous(int64_t seq_num); + + void FrameDecoded(int64_t seq_num); + + void SignalNetworkState(NetworkState state); + + // Returns number of different frames seen. + int GetUniqueFramesSeen() const { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + return frame_counter_.GetUniqueSeen(); + } + + // Implements RtpPacketSinkInterface. + void OnRtpPacket(const RtpPacketReceived& packet) override; + + // TODO(philipel): Stop using VCMPacket in the new jitter buffer and then + // remove this function. Public only for tests. + void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, + const RtpPacketReceived& rtp_packet, + const RTPVideoHeader& video); + + // Implements RecoveredPacketReceiver. + void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; + + // Send an RTCP keyframe request. + void RequestKeyFrame() override; + + // Implements LossNotificationSender. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) override; + + bool IsUlpfecEnabled() const; + bool IsRetransmissionsEnabled() const; + + // Returns true if a decryptor is attached and frames can be decrypted. + // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame + // Decryption not SRTP. + bool IsDecryptable() const; + + // Don't use, still experimental. + void RequestPacketRetransmit(const std::vector& sequence_numbers); + + // Implements OnCompleteFrameCallback. + void OnCompleteFrame( + std::unique_ptr frame) override; + + // Implements OnDecryptedFrameCallback. + void OnDecryptedFrame( + std::unique_ptr frame) override; + + // Implements OnDecryptionStatusChangeCallback. + void OnDecryptionStatusChange( + FrameDecryptorInterface::Status status) override; + + // Optionally set a frame decryptor after a stream has started. This will not + // reset the decoder state. + void SetFrameDecryptor( + rtc::scoped_refptr frame_decryptor); + + // Sets a frame transformer after a stream has started, if no transformer + // has previously been set. Does not reset the decoder state. + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer); + + // Called by VideoReceiveStream when stats are updated. + void UpdateRtt(int64_t max_rtt_ms); + + absl::optional LastReceivedPacketMs() const; + absl::optional LastReceivedKeyframePacketMs() const; + + // RtpDemuxer only forwards a given RTP packet to one sink. However, some + // sinks, such as FlexFEC, might wish to be informed of all of the packets + // a given sink receives (or any set of sinks). They may do so by registering + // themselves as secondary sinks. + void AddSecondarySink(RtpPacketSinkInterface* sink); + void RemoveSecondarySink(const RtpPacketSinkInterface* sink); + + virtual void ManageFrame(std::unique_ptr frame); + + private: + // Used for buffering RTCP feedback messages and sending them all together. + // Note: + // 1. Key frame requests and NACKs are mutually exclusive, with the + // former taking precedence over the latter. + // 2. Loss notifications are orthogonal to either. (That is, may be sent + // alongside either.) + class RtcpFeedbackBuffer : public KeyFrameRequestSender, + public NackSender, + public LossNotificationSender { + public: + RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, + NackSender* nack_sender, + LossNotificationSender* loss_notification_sender); + + ~RtcpFeedbackBuffer() override = default; + + // KeyFrameRequestSender implementation. + void RequestKeyFrame() override; + + // NackSender implementation. + void SendNack(const std::vector& sequence_numbers, + bool buffering_allowed) override; + + // LossNotificationSender implementation. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) override; + + // Send all RTCP feedback messages buffered thus far. + void SendBufferedRtcpFeedback(); + + private: + KeyFrameRequestSender* const key_frame_request_sender_; + NackSender* const nack_sender_; + LossNotificationSender* const loss_notification_sender_; + + // NACKs are accessible from two threads due to nack_module_ being a module. + rtc::CriticalSection cs_; + + // Key-frame-request-related state. + bool request_key_frame_ RTC_GUARDED_BY(cs_); + + // NACK-related state. + std::vector nack_sequence_numbers_ RTC_GUARDED_BY(cs_); + + // LNTF-related state. + struct LossNotificationState { + LossNotificationState(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag) + : last_decoded_seq_num(last_decoded_seq_num), + last_received_seq_num(last_received_seq_num), + decodability_flag(decodability_flag) {} + + uint16_t last_decoded_seq_num; + uint16_t last_received_seq_num; + bool decodability_flag; + }; + absl::optional lntf_state_ RTC_GUARDED_BY(cs_); + }; + enum ParseGenericDependenciesResult { + kDropPacket, + kHasGenericDescriptor, + kNoGenericDescriptor + }; + + // Entry point doing non-stats work for a received packet. Called + // for the same packet both before and after RED decapsulation. + void ReceivePacket(const RtpPacketReceived& packet); + // Parses and handles RED headers. + // This function assumes that it's being called from only one thread. + void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet); + void NotifyReceiverOfEmptyPacket(uint16_t seq_num); + void UpdateHistograms(); + bool IsRedEnabled() const; + void InsertSpsPpsIntoTracker(uint8_t payload_type); + void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result); + ParseGenericDependenciesResult ParseGenericDependenciesExtension( + const RtpPacketReceived& rtp_packet, + RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_); + void OnAssembledFrame(std::unique_ptr frame); + + Clock* const clock_; + // Ownership of this object lies with VideoReceiveStream, which owns |this|. + const VideoReceiveStream::Config& config_; + PacketRouter* const packet_router_; + ProcessThread* const process_thread_; + + RemoteNtpTimeEstimator ntp_estimator_; + + RtpHeaderExtensionMap rtp_header_extensions_; + ReceiveStatistics* const rtp_receive_statistics_; + std::unique_ptr ulpfec_receiver_; + + SequenceChecker worker_task_checker_; + bool receiving_ RTC_GUARDED_BY(worker_task_checker_); + int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); + + const std::unique_ptr rtp_rtcp_; + + video_coding::OnCompleteFrameCallback* complete_frame_callback_; + KeyFrameRequestSender* const keyframe_request_sender_; + + RtcpFeedbackBuffer rtcp_feedback_buffer_; + std::unique_ptr nack_module_; + std::unique_ptr loss_notification_controller_; + + video_coding::PacketBuffer packet_buffer_; + UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_); + SeqNumUnwrapper frame_id_unwrapper_ + RTC_GUARDED_BY(worker_task_checker_); + + // Video structure provided in the dependency descriptor in a first packet + // of a key frame. It is required to parse dependency descriptor in the + // following delta packets. + std::unique_ptr video_structure_ + RTC_GUARDED_BY(worker_task_checker_); + // Frame id of the last frame with the attached video structure. + // absl::nullopt when `video_structure_ == nullptr`; + absl::optional video_structure_frame_id_ + RTC_GUARDED_BY(worker_task_checker_); + + rtc::CriticalSection reference_finder_lock_; + std::unique_ptr reference_finder_ + RTC_GUARDED_BY(reference_finder_lock_); + absl::optional current_codec_; + uint32_t last_assembled_frame_rtp_timestamp_; + + rtc::CriticalSection last_seq_num_cs_; + std::map last_seq_num_for_pic_id_ + RTC_GUARDED_BY(last_seq_num_cs_); + video_coding::H264SpsPpsTracker tracker_; + + // Maps payload id to the depacketizer. + std::map> payload_type_map_; + + +#ifndef DISABLE_H265 + video_coding::H265VpsSpsPpsTracker h265_tracker_; +#endif + + // TODO(johan): Remove pt_codec_params_ once + // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. + // Maps a payload type to a map of out-of-band supplied codec parameters. + std::map> pt_codec_params_; + int16_t last_payload_type_ = -1; + + bool has_received_frame_; + + std::vector secondary_sinks_ + RTC_GUARDED_BY(worker_task_checker_); + + // Info for GetSyncInfo is updated on network or worker thread, and queried on + // the worker thread. + rtc::CriticalSection sync_info_lock_; + absl::optional last_received_rtp_timestamp_ + RTC_GUARDED_BY(sync_info_lock_); + absl::optional last_received_rtp_system_time_ms_ + RTC_GUARDED_BY(sync_info_lock_); + + // Used to validate the buffered frame decryptor is always run on the correct + // thread. + rtc::ThreadChecker network_tc_; + // Handles incoming encrypted frames and forwards them to the + // rtp_reference_finder if they are decryptable. + std::unique_ptr buffered_frame_decryptor_ + RTC_PT_GUARDED_BY(network_tc_); + std::atomic frames_decryptable_; + absl::optional last_color_space_; + + AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_ + RTC_GUARDED_BY(worker_task_checker_); + + int64_t last_completed_picture_id_ = 0; + + rtc::scoped_refptr + frame_transformer_delegate_; +}; + +} // namespace webrtc + +#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index b22f901b7c..bc3b702bdf 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -710,8 +710,9 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData( packet->video_payload = std::move(fixed.bitstream); break; } + } #endif - } else { + else { packet->video_payload = std::move(codec_payload); } diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc new file mode 100644 index 0000000000..f812c8d467 --- /dev/null +++ b/video/video_receive_stream.cc @@ -0,0 +1,801 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/video_receive_stream.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/video/encoded_image.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "call/rtp_stream_receiver_controller_interface.h" +#include "call/rtx_receive_stream.h" +#include "common_video/include/incoming_video_stream.h" +#include "media/base/h264_profile_level_id.h" +#include "modules/utility/include/process_thread.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_coding_defines.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/timing.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/keyframe_interval_settings.h" +#include "rtc_base/location.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/system/thread_registry.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" +#include "video/call_stats.h" +#include "video/frame_dumping_decoder.h" +#include "video/receive_statistics_proxy.h" + +namespace webrtc { + +namespace internal { +constexpr int VideoReceiveStream::kMaxWaitForKeyFrameMs; +} // namespace internal + +namespace { + +using video_coding::EncodedFrame; +using ReturnReason = video_coding::FrameBuffer::ReturnReason; + +constexpr int kMinBaseMinimumDelayMs = 0; +constexpr int kMaxBaseMinimumDelayMs = 10000; + +constexpr int kMaxWaitForFrameMs = 3000; + +// Concrete instance of RecordableEncodedFrame wrapping needed content +// from video_coding::EncodedFrame. +class WebRtcRecordableEncodedFrame : public RecordableEncodedFrame { + public: + explicit WebRtcRecordableEncodedFrame(const EncodedFrame& frame) + : buffer_(frame.GetEncodedData()), + render_time_ms_(frame.RenderTime()), + codec_(frame.CodecSpecific()->codecType), + is_key_frame_(frame.FrameType() == VideoFrameType::kVideoFrameKey), + resolution_{frame.EncodedImage()._encodedWidth, + frame.EncodedImage()._encodedHeight} { + if (frame.ColorSpace()) { + color_space_ = *frame.ColorSpace(); + } + } + + // VideoEncodedSinkInterface::FrameBuffer + rtc::scoped_refptr encoded_buffer() + const override { + return buffer_; + } + + absl::optional color_space() const override { + return color_space_; + } + + VideoCodecType codec() const override { return codec_; } + + bool is_key_frame() const override { return is_key_frame_; } + + EncodedResolution resolution() const override { return resolution_; } + + Timestamp render_time() const override { + return Timestamp::Millis(render_time_ms_); + } + + private: + rtc::scoped_refptr buffer_; + int64_t render_time_ms_; + VideoCodecType codec_; + bool is_key_frame_; + EncodedResolution resolution_; + absl::optional color_space_; +}; + +VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { + VideoCodec codec; + memset(&codec, 0, sizeof(codec)); + + codec.plType = decoder.payload_type; + codec.codecType = PayloadStringToCodecType(decoder.video_format.name); + + if (codec.codecType == kVideoCodecVP8) { + *(codec.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + } else if (codec.codecType == kVideoCodecVP9) { + *(codec.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + } else if (codec.codecType == kVideoCodecH264) { + *(codec.H264()) = VideoEncoder::GetDefaultH264Settings(); + } else if (codec.codecType == kVideoCodecMultiplex) { + VideoReceiveStream::Decoder associated_decoder = decoder; + associated_decoder.video_format = + SdpVideoFormat(CodecTypeToPayloadString(kVideoCodecVP9)); + VideoCodec associated_codec = CreateDecoderVideoCodec(associated_decoder); + associated_codec.codecType = kVideoCodecMultiplex; + return associated_codec; + } +#ifndef DISABLE_H265 + else if (codec.codecType == kVideoCodecH265) { + *(codec.H265()) = VideoEncoder::GetDefaultH265Settings(); + } +#endif + + codec.width = 320; + codec.height = 180; + const int kDefaultStartBitrate = 300; + codec.startBitrate = codec.minBitrate = codec.maxBitrate = + kDefaultStartBitrate; + + return codec; +} + +// Video decoder class to be used for unknown codecs. Doesn't support decoding +// but logs messages to LS_ERROR. +class NullVideoDecoder : public webrtc::VideoDecoder { + public: + int32_t InitDecode(const webrtc::VideoCodec* codec_settings, + int32_t number_of_cores) override { + RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder."; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override { + RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding."; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) override { + RTC_LOG(LS_ERROR) + << "Can't register decode complete callback on NullVideoDecoder."; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } + + const char* ImplementationName() const override { return "NullVideoDecoder"; } +}; + +// TODO(https://bugs.webrtc.org/9974): Consider removing this workaround. +// Maximum time between frames before resetting the FrameBuffer to avoid RTP +// timestamps wraparound to affect FrameBuffer. +constexpr int kInactiveStreamThresholdMs = 600000; // 10 minutes. + +} // namespace + +namespace internal { + +VideoReceiveStream::VideoReceiveStream( + TaskQueueFactory* task_queue_factory, + RtpStreamReceiverControllerInterface* receiver_controller, + int num_cpu_cores, + PacketRouter* packet_router, + VideoReceiveStream::Config config, + ProcessThread* process_thread, + CallStats* call_stats, + Clock* clock, + VCMTiming* timing) + : task_queue_factory_(task_queue_factory), + transport_adapter_(config.rtcp_send_transport), + config_(std::move(config)), + num_cpu_cores_(num_cpu_cores), + process_thread_(process_thread), + clock_(clock), + call_stats_(call_stats), + source_tracker_(clock_), + stats_proxy_(&config_, clock_), + rtp_receive_statistics_(ReceiveStatistics::Create(clock_)), + timing_(timing), + video_receiver_(clock_, timing_.get()), + rtp_video_stream_receiver_(clock_, + &transport_adapter_, + call_stats, + packet_router, + &config_, + rtp_receive_statistics_.get(), + &stats_proxy_, + process_thread_, + this, // NackSender + nullptr, // Use default KeyFrameRequestSender + this, // OnCompleteFrameCallback + config_.frame_decryptor, + config_.frame_transformer), + rtp_stream_sync_(this), + max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() + .MaxWaitForKeyframeMs() + .value_or(kMaxWaitForKeyFrameMs)), + max_wait_for_frame_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() + .MaxWaitForFrameMs() + .value_or(kMaxWaitForFrameMs)), + decode_queue_(task_queue_factory_->CreateTaskQueue( + "DecodingQueue", + TaskQueueFactory::Priority::HIGH)) { + RTC_LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString(); + + RTC_DCHECK(config_.renderer); + RTC_DCHECK(process_thread_); + RTC_DCHECK(call_stats_); + + module_process_sequence_checker_.Detach(); + network_sequence_checker_.Detach(); + + RTC_DCHECK(!config_.decoders.empty()); + std::set decoder_payload_types; + for (const Decoder& decoder : config_.decoders) { + RTC_CHECK(decoder.decoder_factory); + RTC_CHECK(decoder_payload_types.find(decoder.payload_type) == + decoder_payload_types.end()) + << "Duplicate payload type (" << decoder.payload_type + << ") for different decoders."; + decoder_payload_types.insert(decoder.payload_type); + } + + timing_->set_render_delay(config_.render_delay_ms); + + frame_buffer_.reset( + new video_coding::FrameBuffer(clock_, timing_.get(), &stats_proxy_)); + + process_thread_->RegisterModule(&rtp_stream_sync_, RTC_FROM_HERE); + // Register with RtpStreamReceiverController. + media_receiver_ = receiver_controller->CreateReceiver( + config_.rtp.remote_ssrc, &rtp_video_stream_receiver_); + if (config_.rtp.rtx_ssrc) { + rtx_receive_stream_ = std::make_unique( + &rtp_video_stream_receiver_, config.rtp.rtx_associated_payload_types, + config_.rtp.remote_ssrc, rtp_receive_statistics_.get()); + rtx_receiver_ = receiver_controller->CreateReceiver( + config_.rtp.rtx_ssrc, rtx_receive_stream_.get()); + } else { + rtp_receive_statistics_->EnableRetransmitDetection(config.rtp.remote_ssrc, + true); + } +} + +VideoReceiveStream::VideoReceiveStream( + TaskQueueFactory* task_queue_factory, + RtpStreamReceiverControllerInterface* receiver_controller, + int num_cpu_cores, + PacketRouter* packet_router, + VideoReceiveStream::Config config, + ProcessThread* process_thread, + CallStats* call_stats, + Clock* clock) + : VideoReceiveStream(task_queue_factory, + receiver_controller, + num_cpu_cores, + packet_router, + std::move(config), + process_thread, + call_stats, + clock, + new VCMTiming(clock)) {} + +VideoReceiveStream::~VideoReceiveStream() { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + RTC_LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString(); + Stop(); + process_thread_->DeRegisterModule(&rtp_stream_sync_); +} + +void VideoReceiveStream::SignalNetworkState(NetworkState state) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + rtp_video_stream_receiver_.SignalNetworkState(state); +} + +bool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { + return rtp_video_stream_receiver_.DeliverRtcp(packet, length); +} + +void VideoReceiveStream::SetSync(Syncable* audio_syncable) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + rtp_stream_sync_.ConfigureSync(audio_syncable); +} + +void VideoReceiveStream::Start() { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + + if (decoder_running_) { + return; + } + + const bool protected_by_fec = config_.rtp.protected_by_flexfec || + rtp_video_stream_receiver_.IsUlpfecEnabled(); + + frame_buffer_->Start(); + + if (rtp_video_stream_receiver_.IsRetransmissionsEnabled() && + protected_by_fec) { + frame_buffer_->SetProtectionMode(kProtectionNackFEC); + } + + transport_adapter_.Enable(); + rtc::VideoSinkInterface* renderer = nullptr; + if (config_.enable_prerenderer_smoothing) { + incoming_video_stream_.reset(new IncomingVideoStream( + task_queue_factory_, config_.render_delay_ms, this)); + renderer = incoming_video_stream_.get(); + } else { + renderer = this; + } + + for (const Decoder& decoder : config_.decoders) { + std::unique_ptr video_decoder = + decoder.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format, + config_.stream_id); + // If we still have no valid decoder, we have to create a "Null" decoder + // that ignores all calls. The reason we can get into this state is that the + // old decoder factory interface doesn't have a way to query supported + // codecs. + if (!video_decoder) { + video_decoder = std::make_unique(); + } + + std::string decoded_output_file = + field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory"); + // Because '/' can't be used inside a field trial parameter, we use ';' + // instead. + // This is only relevant to WebRTC-DecoderDataDumpDirectory + // field trial. ';' is chosen arbitrary. Even though it's a legal character + // in some file systems, we can sacrifice ability to use it in the path to + // dumped video, since it's developers-only feature for debugging. + absl::c_replace(decoded_output_file, ';', '/'); + if (!decoded_output_file.empty()) { + char filename_buffer[256]; + rtc::SimpleStringBuilder ssb(filename_buffer); + ssb << decoded_output_file << "/webrtc_receive_stream_" + << this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros() + << ".ivf"; + video_decoder = CreateFrameDumpingDecoderWrapper( + std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str())); + } + + video_decoders_.push_back(std::move(video_decoder)); + + video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(), + decoder.payload_type); + VideoCodec codec = CreateDecoderVideoCodec(decoder); + + const bool raw_payload = + config_.rtp.raw_payload_types.count(codec.plType) > 0; + rtp_video_stream_receiver_.AddReceiveCodec( + codec, decoder.video_format.parameters, raw_payload); + RTC_CHECK_EQ(VCM_OK, video_receiver_.RegisterReceiveCodec( + &codec, num_cpu_cores_, false)); + } + + RTC_DCHECK(renderer != nullptr); + video_stream_decoder_.reset( + new VideoStreamDecoder(&video_receiver_, &stats_proxy_, renderer)); + + // Make sure we register as a stats observer *after* we've prepared the + // |video_stream_decoder_|. + call_stats_->RegisterStatsObserver(this); + + // Start decoding on task queue. + video_receiver_.DecoderThreadStarting(); + stats_proxy_.DecoderThreadStarting(); + decode_queue_.PostTask([this] { + RTC_DCHECK_RUN_ON(&decode_queue_); + decoder_stopped_ = false; + StartNextDecode(); + }); + decoder_running_ = true; + rtp_video_stream_receiver_.StartReceive(); +} + +void VideoReceiveStream::Stop() { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + rtp_video_stream_receiver_.StopReceive(); + + stats_proxy_.OnUniqueFramesCounted( + rtp_video_stream_receiver_.GetUniqueFramesSeen()); + + decode_queue_.PostTask([this] { frame_buffer_->Stop(); }); + + call_stats_->DeregisterStatsObserver(this); + + if (decoder_running_) { + rtc::Event done; + decode_queue_.PostTask([this, &done] { + RTC_DCHECK_RUN_ON(&decode_queue_); + decoder_stopped_ = true; + done.Set(); + }); + done.Wait(rtc::Event::kForever); + + decoder_running_ = false; + video_receiver_.DecoderThreadStopped(); + stats_proxy_.DecoderThreadStopped(); + // Deregister external decoders so they are no longer running during + // destruction. This effectively stops the VCM since the decoder thread is + // stopped, the VCM is deregistered and no asynchronous decoder threads are + // running. + for (const Decoder& decoder : config_.decoders) + video_receiver_.RegisterExternalDecoder(nullptr, decoder.payload_type); + + UpdateHistograms(); + } + + video_stream_decoder_.reset(); + incoming_video_stream_.reset(); + transport_adapter_.Disable(); +} + +VideoReceiveStream::Stats VideoReceiveStream::GetStats() const { + VideoReceiveStream::Stats stats = stats_proxy_.GetStats(); + stats.total_bitrate_bps = 0; + StreamStatistician* statistician = + rtp_receive_statistics_->GetStatistician(stats.ssrc); + if (statistician) { + stats.rtp_stats = statistician->GetStats(); + stats.total_bitrate_bps = statistician->BitrateReceived(); + } + if (config_.rtp.rtx_ssrc) { + StreamStatistician* rtx_statistician = + rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc); + if (rtx_statistician) + stats.total_bitrate_bps += rtx_statistician->BitrateReceived(); + } + return stats; +} + +void VideoReceiveStream::UpdateHistograms() { + absl::optional fraction_lost; + StreamDataCounters rtp_stats; + StreamStatistician* statistician = + rtp_receive_statistics_->GetStatistician(config_.rtp.remote_ssrc); + if (statistician) { + fraction_lost = statistician->GetFractionLostInPercent(); + rtp_stats = statistician->GetReceiveStreamDataCounters(); + } + if (config_.rtp.rtx_ssrc) { + StreamStatistician* rtx_statistician = + rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc); + if (rtx_statistician) { + StreamDataCounters rtx_stats = + rtx_statistician->GetReceiveStreamDataCounters(); + stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, &rtx_stats); + return; + } + } + stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, nullptr); +} + +void VideoReceiveStream::AddSecondarySink(RtpPacketSinkInterface* sink) { + rtp_video_stream_receiver_.AddSecondarySink(sink); +} + +void VideoReceiveStream::RemoveSecondarySink( + const RtpPacketSinkInterface* sink) { + rtp_video_stream_receiver_.RemoveSecondarySink(sink); +} + +bool VideoReceiveStream::SetBaseMinimumPlayoutDelayMs(int delay_ms) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + if (delay_ms < kMinBaseMinimumDelayMs || delay_ms > kMaxBaseMinimumDelayMs) { + return false; + } + + rtc::CritScope cs(&playout_delay_lock_); + base_minimum_playout_delay_ms_ = delay_ms; + UpdatePlayoutDelays(); + return true; +} + +int VideoReceiveStream::GetBaseMinimumPlayoutDelayMs() const { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + + rtc::CritScope cs(&playout_delay_lock_); + return base_minimum_playout_delay_ms_; +} + +// TODO(tommi): This method grabs a lock 6 times. +void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) { + int64_t video_playout_ntp_ms; + int64_t sync_offset_ms; + double estimated_freq_khz; + // TODO(tommi): GetStreamSyncOffsetInMs grabs three locks. One inside the + // function itself, another in GetChannel() and a third in + // GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function + // succeeds most of the time, which leads to grabbing a fourth lock. + if (rtp_stream_sync_.GetStreamSyncOffsetInMs( + video_frame.timestamp(), video_frame.render_time_ms(), + &video_playout_ntp_ms, &sync_offset_ms, &estimated_freq_khz)) { + // TODO(tommi): OnSyncOffsetUpdated grabs a lock. + stats_proxy_.OnSyncOffsetUpdated(video_playout_ntp_ms, sync_offset_ms, + estimated_freq_khz); + } + source_tracker_.OnFrameDelivered(video_frame.packet_infos()); + + config_.renderer->OnFrame(video_frame); + + // TODO(tommi): OnRenderFrame grabs a lock too. + stats_proxy_.OnRenderedFrame(video_frame); +} + +void VideoReceiveStream::SetFrameDecryptor( + rtc::scoped_refptr frame_decryptor) { + rtp_video_stream_receiver_.SetFrameDecryptor(std::move(frame_decryptor)); +} + +void VideoReceiveStream::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) { + rtp_video_stream_receiver_.SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); +} + +void VideoReceiveStream::SendNack(const std::vector& sequence_numbers, + bool buffering_allowed) { + RTC_DCHECK(buffering_allowed); + rtp_video_stream_receiver_.RequestPacketRetransmit(sequence_numbers); +} + +void VideoReceiveStream::RequestKeyFrame(int64_t timestamp_ms) { + rtp_video_stream_receiver_.RequestKeyFrame(); + last_keyframe_request_ms_ = timestamp_ms; +} + +void VideoReceiveStream::OnCompleteFrame( + std::unique_ptr frame) { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + // TODO(https://bugs.webrtc.org/9974): Consider removing this workaround. + int64_t time_now_ms = clock_->TimeInMilliseconds(); + if (last_complete_frame_time_ms_ > 0 && + time_now_ms - last_complete_frame_time_ms_ > kInactiveStreamThresholdMs) { + frame_buffer_->Clear(); + } + last_complete_frame_time_ms_ = time_now_ms; + + const PlayoutDelay& playout_delay = frame->EncodedImage().playout_delay_; + if (playout_delay.min_ms >= 0) { + rtc::CritScope cs(&playout_delay_lock_); + frame_minimum_playout_delay_ms_ = playout_delay.min_ms; + UpdatePlayoutDelays(); + } + + if (playout_delay.max_ms >= 0) { + rtc::CritScope cs(&playout_delay_lock_); + frame_maximum_playout_delay_ms_ = playout_delay.max_ms; + UpdatePlayoutDelays(); + } + + int64_t last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame)); + if (last_continuous_pid != -1) + rtp_video_stream_receiver_.FrameContinuous(last_continuous_pid); +} + +void VideoReceiveStream::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) { + RTC_DCHECK_RUN_ON(&module_process_sequence_checker_); + frame_buffer_->UpdateRtt(max_rtt_ms); + rtp_video_stream_receiver_.UpdateRtt(max_rtt_ms); +} + +uint32_t VideoReceiveStream::id() const { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + return config_.rtp.remote_ssrc; +} + +absl::optional VideoReceiveStream::GetInfo() const { + RTC_DCHECK_RUN_ON(&module_process_sequence_checker_); + absl::optional info = + rtp_video_stream_receiver_.GetSyncInfo(); + + if (!info) + return absl::nullopt; + + info->current_delay_ms = timing_->TargetVideoDelay(); + return info; +} + +bool VideoReceiveStream::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, + int64_t* time_ms) const { + RTC_NOTREACHED(); + return 0; +} + +void VideoReceiveStream::SetEstimatedPlayoutNtpTimestampMs( + int64_t ntp_timestamp_ms, + int64_t time_ms) { + RTC_NOTREACHED(); +} + +void VideoReceiveStream::SetMinimumPlayoutDelay(int delay_ms) { + RTC_DCHECK_RUN_ON(&module_process_sequence_checker_); + rtc::CritScope cs(&playout_delay_lock_); + syncable_minimum_playout_delay_ms_ = delay_ms; + UpdatePlayoutDelays(); +} + +int64_t VideoReceiveStream::GetWaitMs() const { + return keyframe_required_ ? max_wait_for_keyframe_ms_ + : max_wait_for_frame_ms_; +} + +void VideoReceiveStream::StartNextDecode() { + TRACE_EVENT0("webrtc", "VideoReceiveStream::StartNextDecode"); + frame_buffer_->NextFrame( + GetWaitMs(), keyframe_required_, &decode_queue_, + /* encoded frame handler */ + [this](std::unique_ptr frame, ReturnReason res) { + RTC_DCHECK_EQ(frame == nullptr, res == ReturnReason::kTimeout); + RTC_DCHECK_EQ(frame != nullptr, res == ReturnReason::kFrameFound); + decode_queue_.PostTask([this, frame = std::move(frame)]() mutable { + RTC_DCHECK_RUN_ON(&decode_queue_); + if (decoder_stopped_) + return; + if (frame) { + HandleEncodedFrame(std::move(frame)); + } else { + HandleFrameBufferTimeout(); + } + StartNextDecode(); + }); + }); +} + +void VideoReceiveStream::HandleEncodedFrame( + std::unique_ptr frame) { + int64_t now_ms = clock_->TimeInMilliseconds(); + + // Current OnPreDecode only cares about QP for VP8. + int qp = -1; + if (frame->CodecSpecific()->codecType == kVideoCodecVP8) { + if (!vp8::GetQp(frame->data(), frame->size(), &qp)) { + RTC_LOG(LS_WARNING) << "Failed to extract QP from VP8 video frame"; + } + } + stats_proxy_.OnPreDecode(frame->CodecSpecific()->codecType, qp); + HandleKeyFrameGeneration(frame->FrameType() == VideoFrameType::kVideoFrameKey, + now_ms); + int decode_result = video_receiver_.Decode(frame.get()); + if (decode_result == WEBRTC_VIDEO_CODEC_OK || + decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) { + keyframe_required_ = false; + frame_decoded_ = true; + rtp_video_stream_receiver_.FrameDecoded(frame->id.picture_id); + + if (decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) + RequestKeyFrame(now_ms); + } else if (!frame_decoded_ || !keyframe_required_ || + (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_ < now_ms)) { + keyframe_required_ = true; + // TODO(philipel): Remove this keyframe request when downstream project + // has been fixed. + RequestKeyFrame(now_ms); + } + + if (encoded_frame_buffer_function_) { + frame->Retain(); + encoded_frame_buffer_function_(WebRtcRecordableEncodedFrame(*frame)); + } +} + +void VideoReceiveStream::HandleKeyFrameGeneration( + bool received_frame_is_keyframe, + int64_t now_ms) { + // Repeat sending keyframe requests if we've requested a keyframe. + if (!keyframe_generation_requested_) { + return; + } + if (received_frame_is_keyframe) { + keyframe_generation_requested_ = false; + } else if (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_ <= now_ms) { + if (!IsReceivingKeyFrame(now_ms)) { + RequestKeyFrame(now_ms); + } + } else { + // It hasn't been long enough since the last keyframe request, do nothing. + } +} + +void VideoReceiveStream::HandleFrameBufferTimeout() { + int64_t now_ms = clock_->TimeInMilliseconds(); + absl::optional last_packet_ms = + rtp_video_stream_receiver_.LastReceivedPacketMs(); + + // To avoid spamming keyframe requests for a stream that is not active we + // check if we have received a packet within the last 5 seconds. + bool stream_is_active = last_packet_ms && now_ms - *last_packet_ms < 5000; + if (!stream_is_active) + stats_proxy_.OnStreamInactive(); + + if (stream_is_active && !IsReceivingKeyFrame(now_ms) && + (!config_.crypto_options.sframe.require_frame_encryption || + rtp_video_stream_receiver_.IsDecryptable())) { + RTC_LOG(LS_WARNING) << "No decodable frame in " << GetWaitMs() + << " ms, requesting keyframe."; + RequestKeyFrame(now_ms); + } +} + +bool VideoReceiveStream::IsReceivingKeyFrame(int64_t timestamp_ms) const { + absl::optional last_keyframe_packet_ms = + rtp_video_stream_receiver_.LastReceivedKeyframePacketMs(); + + // If we recently have been receiving packets belonging to a keyframe then + // we assume a keyframe is currently being received. + bool receiving_keyframe = + last_keyframe_packet_ms && + timestamp_ms - *last_keyframe_packet_ms < max_wait_for_keyframe_ms_; + return receiving_keyframe; +} + +void VideoReceiveStream::UpdatePlayoutDelays() const { + const int minimum_delay_ms = + std::max({frame_minimum_playout_delay_ms_, base_minimum_playout_delay_ms_, + syncable_minimum_playout_delay_ms_}); + if (minimum_delay_ms >= 0) { + timing_->set_min_playout_delay(minimum_delay_ms); + } + + const int maximum_delay_ms = frame_maximum_playout_delay_ms_; + if (maximum_delay_ms >= 0) { + timing_->set_max_playout_delay(maximum_delay_ms); + } +} + +std::vector VideoReceiveStream::GetSources() const { + return source_tracker_.GetSources(); +} + +VideoReceiveStream::RecordingState VideoReceiveStream::SetAndGetRecordingState( + RecordingState state, + bool generate_key_frame) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + rtc::Event event; + RecordingState old_state; + decode_queue_.PostTask([this, &event, &old_state, generate_key_frame, + state = std::move(state)] { + RTC_DCHECK_RUN_ON(&decode_queue_); + // Save old state. + old_state.callback = std::move(encoded_frame_buffer_function_); + old_state.keyframe_needed = keyframe_generation_requested_; + old_state.last_keyframe_request_ms = last_keyframe_request_ms_; + + // Set new state. + encoded_frame_buffer_function_ = std::move(state.callback); + if (generate_key_frame) { + RequestKeyFrame(clock_->TimeInMilliseconds()); + keyframe_generation_requested_ = true; + } else { + keyframe_generation_requested_ = state.keyframe_needed; + last_keyframe_request_ms_ = state.last_keyframe_request_ms.value_or(0); + } + event.Set(); + }); + event.Wait(rtc::Event::kForever); + return old_state; +} + +void VideoReceiveStream::GenerateKeyFrame() { + decode_queue_.PostTask([this]() { + RTC_DCHECK_RUN_ON(&decode_queue_); + RequestKeyFrame(clock_->TimeInMilliseconds()); + keyframe_generation_requested_ = true; + }); +} + +} // namespace internal +} // namespace webrtc