Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

RTMP: Support enhanced RTMP specification for HEVC. v6.0.42 #3495

Merged
merged 9 commits into from
Apr 8, 2023
Merged

RTMP: Support enhanced RTMP specification for HEVC. v6.0.42 #3495

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3a469d2
RTMP: Support enhanced RTMP specification for HEVC.
winlinvip Apr 2, 2023
d5b4e67
Kernel: Refine video avc demux.
chundonglinlin Apr 2, 2023
647c5fa
UTest: Support enhanced RTMP specification for HEVC.
chundonglinlin Apr 2, 2023
15e2e12
UTest: Refine HEVC utest cases.
chundonglinlin Apr 3, 2023
08ea535
Refine code.
winlinvip Apr 5, 2023
6344d15
Add utests.
winlinvip Apr 5, 2023
b5909a3
Add utests.
winlinvip Apr 5, 2023
5152d7c
Player: Upgrade mpegts.js to support HTTP-TS/FLV by publishing enhanc…
chundonglinlin Apr 7, 2023
5d06c01
Update to v6.0.42
winlinvip Apr 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 125 additions & 56 deletions trunk/src/kernel/srs_kernel_codec.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,10 @@ bool SrsFlvVideo::keyframe(char* data, int size)
if (size < 1) {
return false;
}

char frame_type = data[0];

// See rtmp_specification_1.0.pdf
// See https://github.com/veovera/enhanced-rtmp
uint8_t frame_type = data[0] & 0x7f;
frame_type = (frame_type >> 4) & 0x0F;

return frame_type == SrsVideoAvcFrameTypeKeyFrame;
Expand All @@ -173,14 +175,23 @@ bool SrsFlvVideo::sh(char* data, int size)
if (size < 2) {
return false;
}

char frame_type = data[0];
frame_type = (frame_type >> 4) & 0x0F;

char avc_packet_type = data[1];


uint8_t frame_type = data[0];
bool is_ext_header = frame_type & 0x80;
SrsVideoAvcFrameTrait avc_packet_type = SrsVideoAvcFrameTraitForbidden;
if (!is_ext_header) {
// See rtmp_specification_1.0.pdf
frame_type = (frame_type >> 4) & 0x0F;
avc_packet_type = (SrsVideoAvcFrameTrait)data[1];
} else {
// See https://github.com/veovera/enhanced-rtmp
avc_packet_type = (SrsVideoAvcFrameTrait)(frame_type & 0x0f);
frame_type = (frame_type >> 4) & 0x07;
}

// Note that SrsVideoHEVCFrameTraitPacketTypeSequenceStart is equal to SrsVideoAvcFrameTraitSequenceHeader
return frame_type == SrsVideoAvcFrameTypeKeyFrame
&& avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader;
&& avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader;
}

bool SrsFlvVideo::h264(char* data, int size)
Expand All @@ -204,8 +215,24 @@ bool SrsFlvVideo::hevc(char* data, int size)
return false;
}

char codec_id = data[0];
codec_id = codec_id & 0x0F;
uint8_t frame_type = data[0];
bool is_ext_header = frame_type & 0x80;
SrsVideoCodecId codec_id = SrsVideoCodecIdForbidden;
if (!is_ext_header) {
// See rtmp_specification_1.0.pdf
codec_id = (SrsVideoCodecId)(frame_type & 0x0F);
} else {
// See https://github.com/veovera/enhanced-rtmp
if (size < 5) {
return false;
}

// Video FourCC
if (data[1] != 'h' || data[2] != 'v' || data[3] != 'c' || data[4] != '1') {
return false;
}
codec_id = SrsVideoCodecIdHEVC;
}

return codec_id == SrsVideoCodecIdHEVC;
}
Expand All @@ -218,12 +245,34 @@ bool SrsFlvVideo::acceptable(char* data, int size)
return false;
}

char frame_type = data[0];
SrsVideoCodecId codec_id = (SrsVideoCodecId)(uint8_t)(frame_type & 0x0f);
frame_type = (frame_type >> 4) & 0x0f;

if (frame_type < 1 || frame_type > 5) {
return false;
uint8_t frame_type = data[0];
bool is_ext_header = frame_type & 0x80;
SrsVideoCodecId codec_id = SrsVideoCodecIdForbidden;
if (!is_ext_header) {
// See rtmp_specification_1.0.pdf
codec_id = (SrsVideoCodecId)(frame_type & 0x0f);
frame_type = (frame_type >> 4) & 0x0f;

if (frame_type < 1 || frame_type > 5) {
return false;
}
} else {
// See https://github.com/veovera/enhanced-rtmp
uint8_t packet_type = frame_type & 0x0f;
frame_type = (frame_type >> 4) & 0x07;

if (packet_type > SrsVideoHEVCFrameTraitPacketTypeMPEG2TSSequenceStart || frame_type > SrsVideoAvcFrameTypeVideoInfoFrame) {
return false;
}

if (size < 5) {
return false;
}

if (data[1] != 'h' || data[2] != 'v' || data[3] != 'c' || data[4] != '1') {
return false;
}
codec_id = SrsVideoCodecIdHEVC;
}

if (codec_id != SrsVideoCodecIdAVC && codec_id != SrsVideoCodecIdAV1 && codec_id != SrsVideoCodecIdHEVC) {
Expand Down Expand Up @@ -775,33 +824,7 @@ srs_error_t SrsFormat::on_video(int64_t timestamp, char* data, int size)

SrsBuffer* buffer = new SrsBuffer(data, size);
SrsAutoFree(SrsBuffer, buffer);

// We already checked the size is positive and data is not NULL.
srs_assert(buffer->require(1));

// @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78
int8_t frame_type = buffer->read_1bytes();
SrsVideoCodecId codec_id = (SrsVideoCodecId)(frame_type & 0x0f);

// Check codec for H.264 and H.265.
bool codec_ok = (codec_id == SrsVideoCodecIdAVC);
#ifdef SRS_H265
codec_ok = codec_ok ? true : (codec_id == SrsVideoCodecIdHEVC);
#endif
if (!codec_ok) return err;

if (!vcodec) {
vcodec = new SrsVideoCodecConfig();
}
if (!video) {
video = new SrsVideoFrame();
}

if ((err = video->initialize(vcodec)) != srs_success) {
return srs_error_wrap(err, "init video");
}

buffer->skip(-1 * buffer->pos());
return video_avc_demux(buffer, timestamp);
}

Expand Down Expand Up @@ -847,12 +870,46 @@ bool SrsFormat::is_avc_sequence_header()
srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp)
{
srs_error_t err = srs_success;


if (!stream->require(1)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "sps shall atleast 1bytes");
}

// @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78
int8_t frame_type = stream->read_1bytes();
SrsVideoCodecId codec_id = (SrsVideoCodecId)(frame_type & 0x0f);
frame_type = (frame_type >> 4) & 0x0f;

SrsVideoCodecId codec_id = SrsVideoCodecIdForbidden;
SrsVideoAvcFrameTrait avc_packet_type = SrsVideoAvcFrameTraitForbidden;
uint8_t frame_type = stream->read_1bytes();
bool is_ext_header = frame_type & 0x80;
if (!is_ext_header) {
// See rtmp_specification_1.0.pdf
codec_id = (SrsVideoCodecId)(frame_type & 0x0f);
frame_type = (frame_type >> 4) & 0x0f;
} else {
// See https://github.com/veovera/enhanced-rtmp
if (!stream->require(4)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "fourCC requires 4bytes, only %dbytes", stream->left());
}

uint32_t four_cc= stream->read_4bytes();
if (four_cc == 0x68766331) {// 'hvc1'=0x68766331
codec_id = SrsVideoCodecIdHEVC;
}
avc_packet_type = (SrsVideoAvcFrameTrait)(frame_type & 0x0f);
frame_type = (frame_type >> 4) & 0x07;
}

if (!vcodec) {
vcodec = new SrsVideoCodecConfig();
}

if (!video) {
video = new SrsVideoFrame();
}

if ((err = video->initialize(vcodec)) != srs_success) {
return srs_error_wrap(err, "init video");
}

video->frame_type = (SrsVideoAvcFrameType)frame_type;

// ignore info frame without error,
Expand All @@ -871,17 +928,29 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp)
return srs_error_new(ERROR_HLS_DECODE_ERROR, "only support video H.264/H.265, actual=%d", codec_id);
}
vcodec->id = codec_id;

if (!stream->require(4)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc decode avc_packet_type");

int32_t composition_time = 0;
if (!is_ext_header) {
// See rtmp_specification_1.0.pdf
if (!stream->require(4)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "avc_packet_type and composition_time requires 4bytes, only %dbytes", stream->left());
}
avc_packet_type = (SrsVideoAvcFrameTrait)stream->read_1bytes();
composition_time = stream->read_3bytes();
} else {
// See https://github.com/veovera/enhanced-rtmp
if (avc_packet_type == SrsVideoHEVCFrameTraitPacketTypeCodedFrames) {
if (!stream->require(3)) {
return srs_error_new(ERROR_HLS_DECODE_ERROR, "hevc decode cts");
}
composition_time = stream->read_3bytes();
}
}
int8_t avc_packet_type = stream->read_1bytes();
int32_t composition_time = stream->read_3bytes();


// pts = dts + cts.
video->dts = timestamp;
video->cts = composition_time;
video->avc_packet_type = (SrsVideoAvcFrameTrait)avc_packet_type;
video->avc_packet_type = avc_packet_type;

// Update the RAW AVC data.
raw = stream->data() + stream->pos();
Expand All @@ -895,7 +964,7 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp)
if ((err = hevc_demux_hvcc(stream)) != srs_success) {
return srs_error_wrap(err, "demux hevc VPS/SPS/PPS");
}
} else if (avc_packet_type == SrsVideoAvcFrameTraitNALU) {
} else if (avc_packet_type == SrsVideoAvcFrameTraitNALU || avc_packet_type == SrsVideoHEVCFrameTraitPacketTypeCodedFramesX) {
// TODO: demux nalu for hevc
if ((err = video_nalu_demux(stream)) != srs_success) {
return srs_error_wrap(err, "demux hevc NALU");
Expand Down
24 changes: 22 additions & 2 deletions trunk/src/kernel/srs_kernel_codec.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,32 @@ std::string srs_video_codec_id2str(SrsVideoCodecId codec);
enum SrsVideoAvcFrameTrait
{
// set to the max value to reserved, for array map.
SrsVideoAvcFrameTraitReserved = 3,
SrsVideoAvcFrameTraitForbidden = 3,
SrsVideoAvcFrameTraitReserved = 6,
SrsVideoAvcFrameTraitForbidden = 6,

SrsVideoAvcFrameTraitSequenceHeader = 0,
SrsVideoAvcFrameTraitNALU = 1,
SrsVideoAvcFrameTraitSequenceHeaderEOF = 2,

SrsVideoHEVCFrameTraitPacketTypeSequenceStart = 0,
SrsVideoHEVCFrameTraitPacketTypeCodedFrames = 1,
SrsVideoHEVCFrameTraitPacketTypeSequenceEnd = 2,
// CompositionTime Offset is implied to equal zero. This is
// an optimization to save putting SI24 composition time value of zero on
// the wire. See pseudo code below in the VideoTagBody section
SrsVideoHEVCFrameTraitPacketTypeCodedFramesX = 3,
// VideoTagBody does not contain video data. VideoTagBody
// instead contains an AMF encoded metadata. See Metadata Frame
// section for an illustration of its usage. As an example, the metadata
// can be HDR information. This is a good way to signal HDR
// information. This also opens up future ways to express additional
// metadata that is meant for the next video sequence.
//
// note: presence of PacketTypeMetadata means that FrameType
// flags at the top of this table should be ignored
SrsVideoHEVCFrameTraitPacketTypeMetadata = 4,
// Carriage of bitstream in MPEG-2 TS format
SrsVideoHEVCFrameTraitPacketTypeMPEG2TSSequenceStart = 5,
};

/**
Expand Down
80 changes: 78 additions & 2 deletions trunk/src/utest/srs_utest_kernel.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3897,7 +3897,7 @@ VOID TEST(KernelCodecTest, VideoFormat)

HELPER_EXPECT_SUCCESS(f.on_video(0, NULL, 0));
HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)"\x00", 0));
HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)"\x00", 1));
HELPER_EXPECT_FAILED(f.on_video(0, (char*)"\x00", 1));
}

if (true) {
Expand Down Expand Up @@ -4001,7 +4001,12 @@ VOID TEST(KernelCodecTest, HevcVideoFormat)

HELPER_EXPECT_SUCCESS(f.on_video(0, NULL, 0));
HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)"\x00", 0));
HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)"\x00", 1));
HELPER_EXPECT_FAILED(f.on_video(0, (char*)"\x00", 1));

// enhanced rtmp/flv
HELPER_EXPECT_FAILED(f.on_video(0, (char*)"\x80", 1));
HELPER_EXPECT_FAILED(f.on_video(0, (char*)"\x90", 1));
HELPER_EXPECT_FAILED(f.on_video(0, (char*)"\x90\x68\x76\x63\x31", 5));
}

if (true) {
Expand All @@ -4015,6 +4020,14 @@ VOID TEST(KernelCodecTest, HevcVideoFormat)
SrsBuffer b((char*)"\x00", 1);
srs_error_t err = f.video_avc_demux(&b, 0);
HELPER_EXPECT_FAILED(err);

// enhanced rtmp/flv
SrsBuffer b1((char*)"\x80", 1);
HELPER_EXPECT_FAILED(f.video_avc_demux(&b1, 0));
SrsBuffer b2((char*)"\x90", 1);
HELPER_EXPECT_FAILED(f.video_avc_demux(&b2, 0));
SrsBuffer b3((char*)"\x90\x68\x76\x63\x31", 5);
HELPER_EXPECT_FAILED(f.video_avc_demux(&b3, 0));
}

uint8_t vps_sps_pps[] = {
Expand Down Expand Up @@ -4075,6 +4088,69 @@ VOID TEST(KernelCodecTest, HevcVideoFormat)
HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)rawIBMF, sizeof(rawIBMF)));
EXPECT_EQ(1, f.video->nb_samples);
}

// enhanced rtmp
uint8_t ext_vps_sps_pps[] = {
// IsExHeader | FrameType: UB[4]
// PacketType: UB[4]
0x90,
// Video FourCC
0x68, 0x76, 0x63, 0x31,
// SrsHevcDecoderConfigurationRecord
0x01, 0x01, 0x60, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0xf0, 0x00, 0xfc, 0xfd, 0xf8, 0xf8, 0x00, 0x00, 0x0f, 0x03,
// Nalus
// data_byte(1B)+num_nalus(2B)+nal_unit_length(2B)
0x20, 0x00, 0x01, 0x00, 0x18,
// VPS
0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x5d, 0x95, 0x98, 0x09,
// data_byte(1B)+num_nalus(2B)+nal_unit_length(2B)
0x21, 0x00, 0x01, 0x00, 0x28,
// SPS
0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x5d, 0xa0, 0x02, 0x80, 0x80, 0x2d, 0x16,
0x59, 0x59, 0xa4, 0x93, 0x2b, 0xc0, 0x40, 0x40, 0x00, 0x00, 0xfa, 0x40, 0x00, 0x17, 0x70, 0x02,
// data_byte(1B)+num_nalus(2B)+nal_unit_length(2B)
0x22, 0x00, 0x01, 0x00, 0x07,
// PPS
0x44, 0x01, 0xc1, 0x72, 0xb4, 0x62, 0x40
};

uint8_t ext_rawIBMF[] = {
// IsExHeader | FrameType: UB[4]
// PacketType: UB[4]
0x93,
// Video FourCC
0x68, 0x76, 0x63, 0x31,
// HEVC NALU
0x00, 0x00, 0x00, 0x0b,
0x28, 0x1, 0xaf, 0x1d, 0x18, 0x38, 0xd4, 0x38, 0x32, 0xda, 0x23
};

if (true) {
SrsFormat f;
HELPER_EXPECT_SUCCESS(f.initialize());

// firstly demux sequence header
HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)ext_vps_sps_pps, sizeof(ext_vps_sps_pps)));
EXPECT_EQ(1, f.video->frame_type);
EXPECT_EQ(0, f.video->avc_packet_type);
EXPECT_EQ(3, f.vcodec->hevc_dec_conf_record_.nalu_vec.size());
EXPECT_EQ(1280, f.vcodec->width);
EXPECT_EQ(720, f.vcodec->height);

// secondly demux sequence header
HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)ext_vps_sps_pps, sizeof(ext_vps_sps_pps)));
EXPECT_EQ(1, f.video->frame_type);
EXPECT_EQ(0, f.video->avc_packet_type);
EXPECT_EQ(3, f.vcodec->hevc_dec_conf_record_.nalu_vec.size());
EXPECT_EQ(1280, f.vcodec->width);
EXPECT_EQ(720, f.vcodec->height);

HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)ext_rawIBMF, sizeof(ext_rawIBMF)));
EXPECT_EQ(1, f.video->nb_samples);

HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)ext_rawIBMF, sizeof(ext_rawIBMF)));
EXPECT_EQ(1, f.video->nb_samples);
}
}
#endif

Expand Down