external/srs
1
0
Fork 0
mirror of https://github.com/ossrs/srs.git synced 2025-02-14 20:31:56 +00:00
Code Issues Releases Wiki Activity

RTC: Support dropping h.264 SEI from NALUs. v5.0.213 v6.0.125 (#4057)

Browse source 
try to fix #4052.

---------

Co-authored-by: winlin <winlinvip@gmail.com>
This commit is contained in:
Jacob Su 2024-06-03 16:25:49 +08:00 committed by GitHub
parent 282d94d7bb
commit 1656391c67
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 141 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
trunk/conf/full.conf
View file

@ -562,6 +562,13 @@ vhost rtc.vhost.srs.com {
# Overwrite by env SRS_VHOST_RTC_KEEP_BFRAME for all vhosts.
# default: off
keep_bframe off;
# Whether to keep the h.264 SEI type NALU packet.
# DJI drone M30T will send many SEI type NALU packet, while iOS hardware decoder (Video Toolbox)
# dislike to feed it so many SEI NALU between NonIDR and IDR NALU packets.
# @see https://github.com/ossrs/srs/issues/4052
# Overwrite by env SRS_VHOST_RTC_KEEP_AVC_NALU_SEI for all vhosts.
# Default: on
keep_avc_nalu_sei on;
# The transcode audio bitrate, for RTMP to RTC.
# Overwrite by env SRS_VHOST_RTC_OPUS_BITRATE for all vhosts.
# [8000, 320000]

2
trunk/doc/CHANGELOG.md
View file

@ -7,6 +7,7 @@ The changelog for SRS.
<a name="v6-changes"></a>
## SRS 6.0 Changelog
* v6.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v6.0.125 (#4057)
* v6.0, 2024-04-26, Merge [#4044](https://github.com/ossrs/srs/pull/4044): fix: correct SRS_ERRNO_MAP_HTTP duplicate error code. v6.0.124 (#4044)
* v6.0, 2024-04-23, Merge [#4038](https://github.com/ossrs/srs/pull/4038): RTMP: Do not response publish start message if hooks fail. v6.0.123 (#4038)
* v6.0, 2024-04-22, Merge [#4033](https://github.com/ossrs/srs/pull/4033): issue #3967: support x509 certification chiain in single pem file. v6.0.122 (#4033)
@ -135,6 +136,7 @@ The changelog for SRS.
<a name="v5-changes"></a>
## SRS 5.0 Changelog
* v5.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057)
* v5.0, 2024-04-23, Merge [#4038](https://github.com/ossrs/srs/pull/4038): RTMP: Do not response publish start message if hooks fail. v5.0.212 (#4038)
* v5.0, 2024-04-22, Merge [#4033](https://github.com/ossrs/srs/pull/4033): issue #3967: support x509 certification chiain in single pem file. v5.0.211 (#4033)
* v5.0, 2024-03-26, Filter JSONP callback function name. v5.0.210

22
trunk/src/app/srs_app_config.cpp
View file

@ -2726,7 +2726,7 @@ srs_error_t SrsConfig::check_normal_config()
&& m != "bframe" && m != "aac" && m != "stun_timeout" && m != "stun_strict_check"
&& m != "dtls_role" && m != "dtls_version" && m != "drop_for_pt" && m != "rtc_to_rtmp"
&& m != "pli_for_rtmp" && m != "rtmp_to_rtc" && m != "keep_bframe" && m != "opus_bitrate"
&& m != "aac_bitrate") {
&& m != "aac_bitrate" && m != "keep_avc_nalu_sei") {
return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.rtc.%s of %s", m.c_str(), vhost->arg0().c_str());
}
}
@ -4474,6 +4474,26 @@ bool SrsConfig::get_rtc_keep_bframe(string vhost)
return SRS_CONF_PERFER_FALSE(conf->arg0());
}
bool SrsConfig::get_rtc_keep_avc_nalu_sei(std::string vhost)
{
SRS_OVERWRITE_BY_ENV_BOOL2("srs.vhost.rtc.keep_avc_nalu_sei"); // SRS_VHOST_RTC_KEEP_AVC_NALU_SEI
static bool DEFAULT = true;
SrsConfDirective* conf = get_rtc(vhost);
if (!conf) {
return DEFAULT;
}
conf = conf->get("keep_avc_nalu_sei");
if (!conf || conf->arg0().empty()) {
return DEFAULT;
}
return SRS_CONF_PERFER_TRUE(conf->arg0());
}
bool SrsConfig::get_rtc_from_rtmp(string vhost)
{
SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.rtmp_to_rtc"); // SRS_VHOST_RTC_RTMP_TO_RTC

1
trunk/src/app/srs_app_config.hpp
View file

@ -533,6 +533,7 @@ public:
SrsConfDirective* get_rtc(std::string vhost);
bool get_rtc_enabled(std::string vhost);
bool get_rtc_keep_bframe(std::string vhost);
bool get_rtc_keep_avc_nalu_sei(std::string vhost);
bool get_rtc_from_rtmp(std::string vhost);
srs_utime_t get_rtc_stun_timeout(std::string vhost);
bool get_rtc_stun_strict_check(std::string vhost);

36
trunk/src/app/srs_app_rtc_source.cpp
View file

@ -739,6 +739,7 @@ SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc,
codec_ = new SrsAudioTranscoder();
latest_codec_ = SrsAudioCodecIdForbidden;
keep_bframe = false;
keep_avc_nalu_sei = true;
merge_nalus = false;
meta = new SrsMetaCache();
audio_sequence = 0;
@ -771,8 +772,10 @@ srs_error_t SrsRtcRtpBuilder::initialize(SrsRequest* r)
format->try_annexb_first = _srs_config->try_annexb_first(r->vhost);
keep_bframe = _srs_config->get_rtc_keep_bframe(req->vhost);
keep_avc_nalu_sei = _srs_config->get_rtc_keep_avc_nalu_sei(req->vhost);
merge_nalus = _srs_config->get_rtc_server_merge_nalus();
srs_trace("RTC bridge from RTMP, keep_bframe=%d, merge_nalus=%d", keep_bframe, merge_nalus);
srs_trace("RTC bridge from RTMP, keep_bframe=%d, keep_avc_nalu_sei=%d, merge_nalus=%d",
keep_bframe, keep_avc_nalu_sei, merge_nalus);
return err;
}
@ -1013,12 +1016,6 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg)
for (int i = 0; i < nn_samples; i++) {
SrsSample* sample = samples[i];
// We always ignore bframe here, if config to discard bframe,
// the bframe flag will not be set.
if (sample->bframe) {
continue;
}
if (sample->size <= kRtpMaxPayloadSize) {
if ((err = package_single_nalu(msg, sample, pkts)) != srs_success) {
return srs_error_wrap(err, "package single nalu");
@ -1050,14 +1047,27 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format
// Update samples to shared frame.
for (int i = 0; i < format->video->nb_samples; ++i) {
SrsSample* sample = &format->video->samples[i];
if (!keep_avc_nalu_sei && format->vcodec->id == SrsVideoCodecIdAVC) {
SrsAvcNaluType avc_nalu_type;
// TODO: FIXME use static method to parse avc nalu type.
if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) {
return srs_error_wrap(err, "parse avc nalu_type");
}
if (avc_nalu_type == SrsAvcNaluTypeSEI) {
// srs_warn("skip avc nalu type SEI, size=%d", sample->size);
continue;
}
}
// Because RTC does not support B-frame, so we will drop them.
// TODO: Drop B-frame in better way, which not cause picture corruption.
if (!keep_bframe) {
if ((err = sample->parse_bframe()) != srs_success) {
if (!keep_bframe && format->vcodec->id == SrsVideoCodecIdAVC) {
bool is_b_frame;
if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) {
return srs_error_wrap(err, "parse bframe");
}
if (sample->bframe) {
if (is_b_frame) {
continue;
}
}
@ -1137,12 +1147,6 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect
for (int i = 0; i < (int)samples.size(); i++) {
SrsSample* sample = samples[i];
// We always ignore bframe here, if config to discard bframe,
// the bframe flag will not be set.
if (sample->bframe) {
continue;
}
if (!sample->size) {
continue;
}

1
trunk/src/app/srs_app_rtc_source.hpp
View file

@ -262,6 +262,7 @@ private:
SrsAudioCodecId latest_codec_;
SrsAudioTranscoder* codec_;
bool keep_bframe;
bool keep_avc_nalu_sei;
bool merge_nalus;
uint16_t audio_sequence;
uint16_t video_sequence;

2
trunk/src/core/srs_core_version5.hpp
View file

@ -9,6 +9,6 @@
#define VERSION_MAJOR 5
#define VERSION_MINOR 0
#define VERSION_REVISION 212
#define VERSION_REVISION 213
#endif

2
trunk/src/core/srs_core_version6.hpp
View file

@ -9,6 +9,6 @@
#define VERSION_MAJOR 6
#define VERSION_MINOR 0
#define VERSION_REVISION 124
#define VERSION_REVISION 125
#endif

99
trunk/src/kernel/srs_kernel_codec.cpp
View file

@ -504,63 +504,23 @@ SrsSample::SrsSample()
{
size = 0;
bytes = NULL;
bframe = false;
}
SrsSample::SrsSample(char* b, int s)
{
size = s;
bytes = b;
bframe = false;
}
SrsSample::~SrsSample()
{
}
srs_error_t SrsSample::parse_bframe()
{
srs_error_t err = srs_success;
uint8_t header = bytes[0];
SrsAvcNaluType nal_type = (SrsAvcNaluType)(header & kNalTypeMask);
if (nal_type != SrsAvcNaluTypeNonIDR && nal_type != SrsAvcNaluTypeDataPartitionA && nal_type != SrsAvcNaluTypeIDR) {
return err;
}
SrsBuffer* stream = new SrsBuffer(bytes, size);
SrsAutoFree(SrsBuffer, stream);
// Skip nalu header.
stream->skip(1);
SrsBitBuffer bitstream(stream);
int32_t first_mb_in_slice = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}
int32_t slice_type_v = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}
SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v;
if (slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1) {
bframe = true;
srs_verbose("nal_type=%d, slice type=%d", nal_type, slice_type);
}
return err;
}
SrsSample* SrsSample::copy()
{
SrsSample* p = new SrsSample();
p->bytes = bytes;
p->size = size;
p->bframe = bframe;
return p;
}
@ -655,7 +615,6 @@ srs_error_t SrsFrame::add_sample(char* bytes, int size)
SrsSample* sample = &samples[nb_samples++];
sample->bytes = bytes;
sample->size = size;
sample->bframe = false;
return err;
}
@ -739,6 +698,64 @@ SrsVideoCodecConfig* SrsVideoFrame::vcodec()
return (SrsVideoCodecConfig*)codec;
}
srs_error_t SrsVideoFrame::parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type)
{
srs_error_t err = srs_success;
if (sample == NULL || sample->size < 1) {
return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu");
}
uint8_t header = sample->bytes[0];
avc_nalu_type = (SrsAvcNaluType)(header & kNalTypeMask);
return err;
}
srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame)
{
srs_error_t err = srs_success;
if (sample == NULL || sample->size < 1) {
return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu");
}
SrsAvcNaluType nalu_type;
if ((err = parse_avc_nalu_type(sample, nalu_type)) != srs_success) {
return srs_error_wrap(err, "parse avc nalu type error");
}
if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA && nalu_type != SrsAvcNaluTypeIDR) {
is_b_frame = false;
return err;
}
SrsBuffer* stream = new SrsBuffer(sample->bytes, sample->size);
SrsAutoFree(SrsBuffer, stream);
// Skip nalu header.
stream->skip(1);
SrsBitBuffer bitstream(stream);
int32_t first_mb_in_slice = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}
int32_t slice_type_v = 0;
if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) {
return srs_error_wrap(err, "nalu read uev");
}
SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v;
is_b_frame = slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1;
if (is_b_frame) {
srs_verbose("nalu_type=%d, slice type=%d", nalu_type, slice_type);
}
return err;
}
SrsFormat::SrsFormat()
{
acodec = NULL;

9
trunk/src/kernel/srs_kernel_codec.hpp
View file

@ -1120,7 +1120,7 @@ std::string srs_hevc_level2str(SrsHevcLevel level);
/**
* A sample is the unit of frame.
* It's a NALU for H.264.
* It's a NALU for H.264, H.265.
* It's the whole AAC raw data for AAC.
* @remark Neither SPS/PPS or ASC is sample unit, it's codec sequence header.
*/
@ -1131,15 +1131,11 @@ public:
int size;
// The ptr of unit, user must free it.
char* bytes;
// Whether is B frame.
bool bframe;
public:
SrsSample();
SrsSample(char* b, int s);
~SrsSample();
public:
// If we need to know whether sample is bframe, we have to parse the NALU payload.
srs_error_t parse_bframe();
// Copy sample, share the bytes pointer.
SrsSample* copy();
};
@ -1322,6 +1318,9 @@ public:
virtual srs_error_t add_sample(char* bytes, int size);
public:
virtual SrsVideoCodecConfig* vcodec();
public:
static srs_error_t parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type);
static srs_error_t parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame);
};
/**

3
trunk/src/kernel/srs_kernel_error.hpp
View file

@ -276,7 +276,8 @@
XX(ERROR_HEVC_DISABLED , 3098, "HevcDisabled", "HEVC is disabled") \
XX(ERROR_HEVC_DECODE_ERROR , 3099, "HevcDecode", "HEVC decode av stream failed") \
XX(ERROR_MP4_HVCC_CHANGE , 3100, "Mp4HvcCChange", "MP4 does not support video HvcC change") \
XX(ERROR_HEVC_API_NO_PREFIXED , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder")
XX(ERROR_HEVC_API_NO_PREFIXED , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder") \
XX(ERROR_AVC_NALU_EMPTY , 3102, "AvcNaluEmpty", "AVC NALU is empty")
/**************************************************/
/* HTTP/StreamConverter protocol error. */

23
trunk/src/utest/srs_utest_config.cpp
View file

@ -4405,6 +4405,29 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesVhostRtc)
SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "on");
EXPECT_TRUE(conf.get_rtc_keep_bframe("__defaultVhost__"));
{
// make sure the default value is false, if defined incorrect env value.
SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "onn");
EXPECT_FALSE(conf.get_rtc_keep_bframe("__defaultVhost__"));
}
{
SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "off");
EXPECT_FALSE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
}
{
SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "on");
EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
}
{
// make sure the default value is true, if defined incorrect env value.
SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "xx");
EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
}
}
if (true) {