Merge pull request #932 from ZitaLiao/dev/HEVC_support

Add H.265 packetizer and related codec selection

CMakeLists.txt

@@ -79,6 +79,9 @@ set(LIBDATACHANNEL_SOURCES
 	${CMAKE_CURRENT_SOURCE_DIR}/src/h264rtppacketizer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/nalunit.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/h264packetizationhandler.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/src/h265rtppacketizer.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/src/h265nalunit.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/src/h265packetizationhandler.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/av1rtppacketizer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/av1packetizationhandler.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/src/mediachainablehandler.cpp
@@ -117,6 +120,9 @@ set(LIBDATACHANNEL_HEADERS
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264rtppacketizer.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/nalunit.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264packetizationhandler.hpp
+	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265rtppacketizer.hpp
+	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265nalunit.hpp
+	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265packetizationhandler.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1rtppacketizer.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1packetizationhandler.hpp
 	${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/mediachainablehandler.hpp

examples/streamer/main.cpp

@@ -210,7 +210,7 @@ shared_ptr<ClientTrackData> addVideo(const shared_ptr<PeerConnection> pc, const
     // create RTP configuration
     auto rtpConfig = make_shared<RtpPacketizationConfig>(ssrc, cname, payloadType, H264RtpPacketizer::defaultClockRate);
     // create packetizer
-    auto packetizer = make_shared<H264RtpPacketizer>(H264RtpPacketizer::Separator::Length, rtpConfig);
+    auto packetizer = make_shared<H264RtpPacketizer>(NalUnit::Separator::Length, rtpConfig);
     // create H264 handler
     auto h264Handler = make_shared<H264PacketizationHandler>(packetizer);
     // add RTCP SR handler

include/rtc/description.hpp

@@ -248,6 +248,7 @@ class RTC_CPP_EXPORT Description {
 		void addVideoCodec(int payloadType, string codec, optional<string> profile = std::nullopt);
 
 		void addH264Codec(int payloadType, optional<string> profile = DEFAULT_H264_VIDEO_PROFILE);
+		void addH265Codec(int payloadType, optional<string> profile = std::nullopt);
 		void addVP8Codec(int payloadType);
 		void addVP9Codec(int payloadType);
 		void addAV1Codec(int payloadType);

include/rtc/h264rtppacketizer.hpp

@@ -23,19 +23,13 @@ class RTC_CPP_EXPORT H264RtpPacketizer final : public RtpPacketizer,
 	shared_ptr<NalUnits> splitMessage(binary_ptr message);
 	const uint16_t maximumFragmentSize;
 
+using Separator=NalUnit::Separator;
+
 public:
 	/// Default clock rate for H264 in RTP
 	inline static const uint32_t defaultClockRate = 90 * 1000;
 
-	/// NAL unit separator
-	enum class Separator {
-		Length = RTC_NAL_SEPARATOR_LENGTH, // first 4 bytes are NAL unit length
-		LongStartSequence = RTC_NAL_SEPARATOR_LONG_START_SEQUENCE,   // 0x00, 0x00, 0x00, 0x01
-		ShortStartSequence = RTC_NAL_SEPARATOR_SHORT_START_SEQUENCE, // 0x00, 0x00, 0x01
-		StartSequence = RTC_NAL_SEPARATOR_START_SEQUENCE, // LongStartSequence or ShortStartSequence
-	};
-
-	H264RtpPacketizer(H264RtpPacketizer::Separator separator,
+	H264RtpPacketizer(Separator separator,
 	                  shared_ptr<RtpPacketizationConfig> rtpConfig,
 	                  uint16_t maximumFragmentSize = NalUnits::defaultMaximumFragmentSize);
 

include/rtc/h265nalunit.hpp

@@ -0,0 +1,181 @@
+/**
+ * Copyright (c) 2023 Zita Liao (Dolby)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef RTC_H265_NAL_UNIT_H
+#define RTC_H265_NAL_UNIT_H
+
+#if RTC_ENABLE_MEDIA
+
+#include "common.hpp"
+#include "nalunit.hpp"
+
+#include <cassert>
+
+namespace rtc {
+
+#pragma pack(push, 1)
+
+#define H265_FU_HEADER_SIZE 1
+/// Nalu header
+struct RTC_CPP_EXPORT H265NalUnitHeader {
+/*
+* nal_unit_header( ) { 
+* forbidden_zero_bit	f(1)
+* nal_unit_type			u(6)
+* nuh_layer_id			u(6)
+* nuh_temporal_id_plus1	u(3)
+}
+*/
+	uint8_t _first = 0; // high byte of header
+	uint8_t _second = 0; // low byte of header
+
+	bool forbiddenBit() const { return _first >> 7; }
+	uint8_t unitType() const { return (_first & 0b0111'1110) >> 1; }
+	uint8_t nuhLayerId() const { return ((_first & 0x1) << 5) | ((_second & 0b1111'1000) >> 3); }
+	uint8_t nuhTempIdPlus1() const { return _second & 0b111;}
+
+	void setForbiddenBit(bool isSet) { _first = (_first & 0x7F) | (isSet << 7); }
+	void setUnitType(uint8_t type) { _first = (_first & 0b1000'0001) | ((type & 0b11'1111) << 1); }
+	void setNuhLayerId(uint8_t nuhLayerId) { 
+			_first = (_first & 0b1111'1110) | ((nuhLayerId & 0b10'0000) >> 5);
+			_second = (_second & 0b0000'0111)	| ((nuhLayerId & 0b01'1111) << 3); }
+	void setNuhTempIdPlus1(uint8_t nuhTempIdPlus1) { _second = (_second & 0b1111'1000) | (nuhTempIdPlus1 & 0b111); }
+};
+
+/// Nalu fragment header
+struct RTC_CPP_EXPORT H265NalUnitFragmentHeader {
+	/*
+	* +---------------+
+	* |0|1|2|3|4|5|6|7|
+	* +-+-+-+-+-+-+-+-+
+	* |S|E|  FuType   |
+	* +---------------+
+	*/
+	uint8_t _first = 0;
+
+	bool isStart() const { return _first >> 7; }
+	bool isEnd() const { return (_first >> 6) & 0x01; }
+	uint8_t unitType() const { return _first & 0b11'1111; }
+
+	void setStart(bool isSet) { _first = (_first & 0x7F) | (isSet << 7); }
+	void setEnd(bool isSet) { _first = (_first & 0b1011'1111) | (isSet << 6); }
+	void setUnitType(uint8_t type) { _first = (_first & 0b1100'0000) | (type & 0b11'1111); }
+};
+
+#pragma pack(pop)
+
+/// Nal unit
+struct RTC_CPP_EXPORT H265NalUnit : NalUnit {
+	H265NalUnit(const H265NalUnit &unit) = default;
+	H265NalUnit(size_t size, bool includingHeader = true) : NalUnit(size, includingHeader, NalUnit::Type::H265) {}
+	H265NalUnit(binary &&data) : NalUnit(std::move(data)) {}
+	H265NalUnit() : NalUnit(NalUnit::Type::H265) {}
+
+	template <typename Iterator> H265NalUnit(Iterator begin_, Iterator end_) : NalUnit(begin_, end_) {}
+
+	bool forbiddenBit() const { return header()->forbiddenBit(); }
+	uint8_t unitType() const { return header()->unitType(); }
+	uint8_t nuhLayerId() const { return header()->nuhLayerId(); }
+	uint8_t nuhTempIdPlus1() const { return header()->nuhTempIdPlus1();}
+
+	binary payload() const {
+		assert(size() >= H265_NAL_HEADER_SIZE);
+		return {begin() + H265_NAL_HEADER_SIZE, end()};
+	}
+
+	void setForbiddenBit(bool isSet) { header()->setForbiddenBit(isSet); }
+	void setUnitType(uint8_t type) { header()->setUnitType(type); }
+	void setNuhLayerId(uint8_t nuhLayerId) { header()->setNuhLayerId(nuhLayerId); }
+	void setNuhTempIdPlus1(uint8_t nuhTempIdPlus1) { header()->setNuhTempIdPlus1(nuhTempIdPlus1); }
+
+	void setPayload(binary payload) {
+		assert(size() >= H265_NAL_HEADER_SIZE);
+		erase(begin() + H265_NAL_HEADER_SIZE, end());
+		insert(end(), payload.begin(), payload.end());
+	}
+
+protected:
+	const H265NalUnitHeader *header() const {
+		assert(size() >= H265_NAL_HEADER_SIZE);
+		return reinterpret_cast<const H265NalUnitHeader *>(data());
+	}
+
+	H265NalUnitHeader *header() {
+		assert(size() >= H265_NAL_HEADER_SIZE);
+		return reinterpret_cast<H265NalUnitHeader *>(data());
+	}
+};
+
+/// Nal unit fragment A
+struct RTC_CPP_EXPORT H265NalUnitFragment : H265NalUnit {
+	static std::vector<shared_ptr<H265NalUnitFragment>> fragmentsFrom(shared_ptr<H265NalUnit> nalu,
+	                                                               uint16_t maximumFragmentSize);
+
+	enum class FragmentType { Start, Middle, End };
+
+	H265NalUnitFragment(FragmentType type, bool forbiddenBit, uint8_t nuhLayerId,
+					 uint8_t nuhTempIdPlus1, uint8_t unitType, binary data);
+
+	uint8_t unitType() const { return fragmentHeader()->unitType(); }
+
+	binary payload() const {
+		assert(size() >= H265_NAL_HEADER_SIZE + H265_FU_HEADER_SIZE);
+		return {begin() + H265_NAL_HEADER_SIZE + H265_FU_HEADER_SIZE, end()};
+	}
+
+	FragmentType type() const {
+		if (fragmentHeader()->isStart()) {
+			return FragmentType::Start;
+		} else if (fragmentHeader()->isEnd()) {
+			return FragmentType::End;
+		} else {
+			return FragmentType::Middle;
+		}
+	}
+
+	void setUnitType(uint8_t type) { fragmentHeader()->setUnitType(type); }
+
+	void setPayload(binary payload) {
+		assert(size() >= H265_NAL_HEADER_SIZE + H265_FU_HEADER_SIZE);
+		erase(begin() + H265_NAL_HEADER_SIZE + H265_FU_HEADER_SIZE, end());
+		insert(end(), payload.begin(), payload.end());
+	}
+
+	void setFragmentType(FragmentType type);
+
+protected:
+	const uint8_t nal_type_fu = 49;
+
+	H265NalUnitHeader *fragmentIndicator() { return reinterpret_cast<H265NalUnitHeader *>(data()); }
+
+	const H265NalUnitHeader *fragmentIndicator() const {
+		return reinterpret_cast<const H265NalUnitHeader *>(data());
+	}
+
+	H265NalUnitFragmentHeader *fragmentHeader() {
+		return reinterpret_cast<H265NalUnitFragmentHeader *>(fragmentIndicator() + H265_NAL_HEADER_SIZE);
+	}
+
+	const H265NalUnitFragmentHeader *fragmentHeader() const {
+		return reinterpret_cast<const H265NalUnitFragmentHeader *>(fragmentIndicator() + H265_NAL_HEADER_SIZE);
+	}
+};
+
+class RTC_CPP_EXPORT H265NalUnits : public std::vector<shared_ptr<H265NalUnit>> {
+public:
+	static const uint16_t defaultMaximumFragmentSize =
+	    uint16_t(RTC_DEFAULT_MTU - 12 - 8 - 40); // SRTP/UDP/IPv6
+
+	std::vector<shared_ptr<binary>> generateFragments(uint16_t maximumFragmentSize);
+};
+
+} // namespace rtc
+
+#endif /* RTC_ENABLE_MEDIA */
+
+#endif /* RTC_NAL_UNIT_H */

include/rtc/h265packetizationhandler.hpp

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2023 Zita Liao (Dolby)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef RTC_H265_PACKETIZATION_HANDLER_H
+#define RTC_H265_PACKETIZATION_HANDLER_H
+
+#if RTC_ENABLE_MEDIA
+
+#include "h265rtppacketizer.hpp"
+#include "mediachainablehandler.hpp"
+#include "h265nalunit.hpp"
+
+namespace rtc {
+
+/// Handler for H265 packetization
+class RTC_CPP_EXPORT H265PacketizationHandler final : public MediaChainableHandler {
+public:
+	/// Construct handler for H265 packetization.
+	/// @param packetizer RTP packetizer for h265
+	H265PacketizationHandler(shared_ptr<H265RtpPacketizer> packetizer);
+};
+
+} // namespace rtc
+
+#endif /* RTC_ENABLE_MEDIA */
+
+#endif /* RTC_H265_PACKETIZATION_HANDLER_H */

include/rtc/h265rtppacketizer.hpp

@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) 2023 Zita Liao (Dolby)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef RTC_H265_RTP_PACKETIZER_H
+#define RTC_H265_RTP_PACKETIZER_H
+
+#if RTC_ENABLE_MEDIA
+
+#include "mediahandlerrootelement.hpp"
+#include "h265nalunit.hpp"
+#include "rtppacketizer.hpp"
+
+namespace rtc {
+
+/// RTP packetization of h265 payload
+class RTC_CPP_EXPORT H265RtpPacketizer final : public RtpPacketizer,
+                                               public MediaHandlerRootElement {
+	shared_ptr<H265NalUnits> splitMessage(binary_ptr message);
+	const uint16_t maximumFragmentSize;
+
+public:
+	/// Default clock rate for H265 in RTP
+	inline static const uint32_t defaultClockRate = 90 * 1000;
+
+	H265RtpPacketizer(NalUnit::Separator separator,
+	                  shared_ptr<RtpPacketizationConfig> rtpConfig,
+	                  uint16_t maximumFragmentSize = H265NalUnits::defaultMaximumFragmentSize);
+
+	/// Constructs h265 payload packetizer with given RTP configuration.
+	/// @note RTP configuration is used in packetization process which may change some configuration
+	/// properties such as sequence number.
+	/// @param rtpConfig  RTP configuration
+	/// @param maximumFragmentSize maximum size of one NALU fragment
+	H265RtpPacketizer(shared_ptr<RtpPacketizationConfig> rtpConfig,
+	                  uint16_t maximumFragmentSize = H265NalUnits::defaultMaximumFragmentSize);
+
+	ChainedOutgoingProduct processOutgoingBinaryMessage(ChainedMessagesProduct messages,
+	                                                    message_ptr control) override;
+
+private:
+	const NalUnit::Separator separator;
+};
+
+} // namespace rtc
+
+#endif /* RTC_ENABLE_MEDIA */
+
+#endif /* RTC_H265_RTP_PACKETIZER_H */