RTC: Support dropping h.264 SEI from NALUs. v5.0.213 v6.0.125 (#4057)

try to fix #4052.

---------

Co-authored-by: winlin <winlinvip@gmail.com>

trunk/src/app/srs_app_config.cpp

@@ -2726,7 +2726,7 @@ srs_error_t SrsConfig::check_normal_config()
                         && m != "bframe" && m != "aac" && m != "stun_timeout" && m != "stun_strict_check"
                         && m != "dtls_role" && m != "dtls_version" && m != "drop_for_pt" && m != "rtc_to_rtmp"
                         && m != "pli_for_rtmp" && m != "rtmp_to_rtc" && m != "keep_bframe" && m != "opus_bitrate"
-                        && m != "aac_bitrate") {
+                        && m != "aac_bitrate" && m != "keep_avc_nalu_sei") {
                         return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.rtc.%s of %s", m.c_str(), vhost->arg0().c_str());
                     }
                 }
@@ -4474,6 +4474,26 @@ bool SrsConfig::get_rtc_keep_bframe(string vhost)
     return SRS_CONF_PERFER_FALSE(conf->arg0());
 }
 
+bool SrsConfig::get_rtc_keep_avc_nalu_sei(std::string vhost)
+{
+    SRS_OVERWRITE_BY_ENV_BOOL2("srs.vhost.rtc.keep_avc_nalu_sei"); // SRS_VHOST_RTC_KEEP_AVC_NALU_SEI
+
+    static bool DEFAULT = true;
+
+    SrsConfDirective* conf = get_rtc(vhost);
+
+    if (!conf) {
+        return DEFAULT;
+    }
+
+    conf = conf->get("keep_avc_nalu_sei");
+    if (!conf || conf->arg0().empty()) {
+        return DEFAULT;
+    }
+
+    return SRS_CONF_PERFER_TRUE(conf->arg0());
+}
+
 bool SrsConfig::get_rtc_from_rtmp(string vhost)
 {
     SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.rtmp_to_rtc"); // SRS_VHOST_RTC_RTMP_TO_RTC

trunk/src/app/srs_app_config.hpp

@@ -533,6 +533,7 @@ public:
     SrsConfDirective* get_rtc(std::string vhost);
     bool get_rtc_enabled(std::string vhost);
     bool get_rtc_keep_bframe(std::string vhost);
+    bool get_rtc_keep_avc_nalu_sei(std::string vhost);
     bool get_rtc_from_rtmp(std::string vhost);
     srs_utime_t get_rtc_stun_timeout(std::string vhost);
     bool get_rtc_stun_strict_check(std::string vhost);

trunk/src/app/srs_app_rtc_source.cpp

@@ -739,6 +739,7 @@ SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc,
     codec_ = new SrsAudioTranscoder();
     latest_codec_ = SrsAudioCodecIdForbidden;
     keep_bframe = false;
+    keep_avc_nalu_sei = true;
     merge_nalus = false;
     meta = new SrsMetaCache();
     audio_sequence = 0;
@@ -771,8 +772,10 @@ srs_error_t SrsRtcRtpBuilder::initialize(SrsRequest* r)
     format->try_annexb_first = _srs_config->try_annexb_first(r->vhost);
 
     keep_bframe = _srs_config->get_rtc_keep_bframe(req->vhost);
+    keep_avc_nalu_sei = _srs_config->get_rtc_keep_avc_nalu_sei(req->vhost);
     merge_nalus = _srs_config->get_rtc_server_merge_nalus();
-    srs_trace("RTC bridge from RTMP, keep_bframe=%d, merge_nalus=%d", keep_bframe, merge_nalus);
+    srs_trace("RTC bridge from RTMP, keep_bframe=%d, keep_avc_nalu_sei=%d, merge_nalus=%d",
+        keep_bframe, keep_avc_nalu_sei, merge_nalus);
 
     return err;
 }
@@ -1013,12 +1016,6 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg)
         for (int i = 0; i < nn_samples; i++) {
             SrsSample* sample = samples[i];
 
-            // We always ignore bframe here, if config to discard bframe,
-            // the bframe flag will not be set.
-            if (sample->bframe) {
-                continue;
-            }
-
             if (sample->size <= kRtpMaxPayloadSize) {
                 if ((err = package_single_nalu(msg, sample, pkts)) != srs_success) {
                     return srs_error_wrap(err, "package single nalu");
@@ -1050,14 +1047,27 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format
     // Update samples to shared frame.
     for (int i = 0; i < format->video->nb_samples; ++i) {
         SrsSample* sample = &format->video->samples[i];
+        
+        if (!keep_avc_nalu_sei && format->vcodec->id == SrsVideoCodecIdAVC) {
+            SrsAvcNaluType avc_nalu_type;
+            // TODO: FIXME use static method to parse avc nalu type.
+            if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) {
+                return srs_error_wrap(err, "parse avc nalu_type");
+            }
+            if (avc_nalu_type == SrsAvcNaluTypeSEI) {
+                // srs_warn("skip avc nalu type SEI, size=%d", sample->size);
+                continue;
+            }
+        }
 
         // Because RTC does not support B-frame, so we will drop them.
         // TODO: Drop B-frame in better way, which not cause picture corruption.
-        if (!keep_bframe) {
-            if ((err = sample->parse_bframe()) != srs_success) {
+        if (!keep_bframe && format->vcodec->id == SrsVideoCodecIdAVC) {
+            bool is_b_frame;
+            if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) {
                 return srs_error_wrap(err, "parse bframe");
             }
-            if (sample->bframe) {
+            if (is_b_frame) {
                 continue;
             }
         }
@@ -1137,12 +1147,6 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect
     for (int i = 0; i < (int)samples.size(); i++) {
         SrsSample* sample = samples[i];
 
-        // We always ignore bframe here, if config to discard bframe,
-        // the bframe flag will not be set.
-        if (sample->bframe) {
-            continue;
-        }
-
         if (!sample->size) {
             continue;
         }

trunk/src/app/srs_app_rtc_source.hpp

@@ -262,6 +262,7 @@ private:
     SrsAudioCodecId latest_codec_;
     SrsAudioTranscoder* codec_;
     bool keep_bframe;
+    bool keep_avc_nalu_sei;
     bool merge_nalus;
     uint16_t audio_sequence;
     uint16_t video_sequence;