SRS5: MP3: Support dump stream information. v5.0.117 (#296) (#3339)

PICK 95defe6dad

trunk/doc/CHANGELOG.md

@@ -21,6 +21,7 @@ The changelog for SRS.
 
 ## SRS 5.0 Changelog
 
+* v5.0, 2022-12-25, For [#296](https://github.com/ossrs/srs/issues/296): MP3: Support dump stream information. v5.0.117
 * v5.0, 2022-12-25, For [#296](https://github.com/ossrs/srs/issues/296): MP3: Support mp3 for RTMP/HLS/HTTP-FLV/HTTP-TS/HLS etc. v5.0.116
 * v5.0, 2022-12-24, Fix [#3328](https://github.com/ossrs/srs/issues/3328): Docker: Avoiding duplicated copy files. v5.0.115
 * v5.0, 2022-12-20, Merge [#3321](https://github.com/ossrs/srs/pull/3321): GB: Refine lazy object GC. v5.0.114

trunk/src/app/srs_app_dvr.cpp

@@ -462,7 +462,7 @@ srs_error_t SrsDvrMp4Segmenter::encode_audio(SrsSharedPtrMessage* audio, SrsForm
     SrsAudioChannels channels = format->acodec->sound_type;
     
     SrsAudioAacFrameTrait ct = format->audio->aac_packet_type;
-    if (ct == SrsAudioAacFrameTraitSequenceHeader || ct == SrsAudioMp3FrameTrait) {
+    if (ct == SrsAudioAacFrameTraitSequenceHeader || ct == SrsAudioMp3FrameTraitSequenceHeader) {
         enc->acodec = sound_format;
         enc->sample_rate = sound_rate;
         enc->sound_bits = sound_size;

trunk/src/app/srs_app_source.cpp

@@ -834,7 +834,6 @@ SrsOriginHub::SrsOriginHub()
     hds = new SrsHds();
 #endif
     ng_exec = new SrsNgExec();
-    format = new SrsRtmpFormat();
     
     _srs_config->subscribe(this);
 }
@@ -853,7 +852,6 @@ SrsOriginHub::~SrsOriginHub()
     }
     srs_freep(ng_exec);
 
-    srs_freep(format);
     srs_freep(hls);
     srs_freep(dash);
     srs_freep(dvr);
@@ -870,13 +868,6 @@ srs_error_t SrsOriginHub::initialize(SrsLiveSource* s, SrsRequest* r)
     req_ = r;
     source = s;
     
-    if ((err = format->initialize()) != srs_success) {
-        return srs_error_wrap(err, "format initialize");
-    }
-
-    // Setup the SPS/PPS parsing strategy.
-    format->try_annexb_first = _srs_config->try_annexb_first(r->vhost);
-    
     if ((err = hls->initialize(this, req_)) != srs_success) {
         return srs_error_wrap(err, "hls initialize");
     }
@@ -922,10 +913,6 @@ srs_error_t SrsOriginHub::on_meta_data(SrsSharedPtrMessage* shared_metadata, Srs
 {
     srs_error_t err = srs_success;
     
-    if ((err = format->on_metadata(packet)) != srs_success) {
-        return srs_error_wrap(err, "Format parse metadata");
-    }
-    
     // copy to all forwarders
     if (true) {
         std::vector<SrsForwarder*>::iterator it;
@@ -949,21 +936,10 @@ srs_error_t SrsOriginHub::on_audio(SrsSharedPtrMessage* shared_audio)
     srs_error_t err = srs_success;
     
     SrsSharedPtrMessage* msg = shared_audio;
+    SrsRtmpFormat* format = source->format_;
     
-    // TODO: FIXME: Support parsing OPUS for RTC.
+    // Handle the metadata when got sequence header.
-    if ((err = format->on_audio(msg)) != srs_success) {
+    if (format->is_aac_sequence_header() || format->is_mp3_sequence_header()) {
-        return srs_error_wrap(err, "format consume audio");
-    }
-
-    // Ignore if no format->acodec, it means the codec is not parsed, or unsupport/unknown codec
-    // such as G.711 codec
-    if (!format->acodec) {
-        return err;
-    }
-    
-    // cache the sequence header if aac
-    // donot cache the sequence header to gop_cache, return here.
-    if (format->is_aac_sequence_header()) {
         srs_assert(format->acodec);
         SrsAudioCodecConfig* c = format->acodec;
         
@@ -972,16 +948,22 @@ srs_error_t SrsOriginHub::on_audio(SrsSharedPtrMessage* shared_audio)
         
         // when got audio stream info.
         SrsStatistic* stat = SrsStatistic::instance();
-        if ((err = stat->on_audio_info(req_, SrsAudioCodecIdAAC, c->sound_rate, c->sound_type, c->aac_object)) != srs_success) {
+        if ((err = stat->on_audio_info(req_, format->acodec->id, c->sound_rate, c->sound_type, c->aac_object)) != srs_success) {
             return srs_error_wrap(err, "stat audio");
         }
 
+        if (format->acodec->id == SrsAudioCodecIdMP3) {
+            srs_trace("%dB audio sh, codec(%d, %dbits, %dchannels, %dHZ)",
+                msg->size, c->id, flv_sample_sizes[c->sound_size], flv_sound_types[c->sound_type],
+                srs_flv_srates[c->sound_rate]);
+        } else {
             srs_trace("%dB audio sh, codec(%d, profile=%s, %dchannels, %dkbps, %dHZ), flv(%dbits, %dchannels, %dHZ)",
                 msg->size, c->id, srs_aac_object2str(c->aac_object).c_str(), c->aac_channels,
                 c->audio_data_rate / 1000, srs_aac_srates[c->aac_sample_rate],
                 flv_sample_sizes[c->sound_size], flv_sound_types[c->sound_type],
                 srs_flv_srates[c->sound_rate]);
         }
+    }
     
     if ((err = hls->on_audio(msg, format)) != srs_success) {
         // apply the error strategy for hls.
@@ -1041,22 +1023,7 @@ srs_error_t SrsOriginHub::on_video(SrsSharedPtrMessage* shared_video, bool is_se
     srs_error_t err = srs_success;
     
     SrsSharedPtrMessage* msg = shared_video;
-    
+    SrsRtmpFormat* format = source->format_;
-    // user can disable the sps parse to workaround when parse sps failed.
-    // @see https://github.com/ossrs/srs/issues/474
-    if (is_sequence_header) {
-        format->avc_parse_sps = _srs_config->get_parse_sps(req_->vhost);
-    }
-    
-    if ((err = format->on_video(msg)) != srs_success) {
-        return srs_error_wrap(err, "format consume video");
-    }
-   
-    // Ignore if no format->vcodec, it means the codec is not parsed, or unsupport/unknown codec
-    // such as H.263 codec
-    if (!format->vcodec) {
-        return err;
-    }
  
     // cache the sequence header if h264
     // donot cache the sequence header to gop_cache, return here.
@@ -1307,6 +1274,8 @@ srs_error_t SrsOriginHub::on_reload_vhost_dash(string vhost)
         return srs_error_wrap(err, "dash start publish");
     }
 
+    SrsRtmpFormat* format = source->format_;
+    
     SrsSharedPtrMessage* cache_sh_video = source->meta->vsh();
     if (cache_sh_video) {
         if ((err = format->on_video(cache_sh_video)) != srs_success) {
@@ -1352,6 +1321,8 @@ srs_error_t SrsOriginHub::on_reload_vhost_hls(string vhost)
     }
     srs_trace("vhost %s hls reload success", vhost.c_str());
 
+    SrsRtmpFormat* format = source->format_;
+    
     // when publish, don't need to fetch sequence header, which is old and maybe corrupt.
     // when reload, we must fetch the sequence header from source cache.
     // notice the source to get the cached sequence header.
@@ -1675,9 +1646,11 @@ srs_error_t SrsMetaCache::dumps(SrsLiveConsumer* consumer, bool atc, SrsRtmpJitt
     // copy sequence header
     // copy audio sequence first, for hls to fast parse the "right" audio codec.
     // @see https://github.com/ossrs/srs/issues/301
+    if (aformat && aformat->acodec && aformat->acodec->id != SrsAudioCodecIdMP3) {
         if (ds && audio && (err = consumer->enqueue(audio, atc, ag)) != srs_success) {
             return srs_error_wrap(err, "enqueue audio sh");
         }
+    }
     
     if (ds && video && (err = consumer->enqueue(video, atc, ag)) != srs_success) {
         return srs_error_wrap(err, "enqueue video sh");
@@ -1963,6 +1936,7 @@ SrsLiveSource::SrsLiveSource()
     gop_cache = new SrsGopCache();
     hub = new SrsOriginHub();
     meta = new SrsMetaCache();
+    format_ = new SrsRtmpFormat();
     
     is_monotonically_increase = false;
     last_packet_time = 0;
@@ -1979,6 +1953,7 @@ SrsLiveSource::~SrsLiveSource()
     // for all consumers are auto free.
     consumers.clear();
 
+    srs_freep(format_);
     srs_freep(hub);
     srs_freep(meta);
     srs_freep(mix_queue);
@@ -2044,6 +2019,13 @@ srs_error_t SrsLiveSource::initialize(SrsRequest* r, ISrsLiveSourceHandler* h)
     req = r->copy();
     atc = _srs_config->get_atc(req->vhost);
 
+    if ((err = format_->initialize()) != srs_success) {
+        return srs_error_wrap(err, "format initialize");
+    }
+
+    // Setup the SPS/PPS parsing strategy.
+    format_->try_annexb_first = _srs_config->try_annexb_first(r->vhost);
+    
     if ((err = hub->initialize(this, req)) != srs_success) {
         return srs_error_wrap(err, "hub");
     }
@@ -2212,6 +2194,10 @@ srs_error_t SrsLiveSource::on_meta_data(SrsCommonMessage* msg, SrsOnMetaDataPack
 {
     srs_error_t err = srs_success;
 
+    if ((err = format_->on_metadata(metadata)) != srs_success) {
+        return srs_error_wrap(err, "Format parse metadata");
+    }
+    
     // if allow atc_auto and bravo-atc detected, open atc for vhost.
     SrsAmf0Any* prop = NULL;
     atc = _srs_config->get_atc(req->vhost);
@@ -2303,8 +2289,19 @@ srs_error_t SrsLiveSource::on_audio_imp(SrsSharedPtrMessage* msg)
 {
     srs_error_t err = srs_success;
 
-    bool is_aac_sequence_header = SrsFlvAudio::sh(msg->payload, msg->size);
+    // TODO: FIXME: Support parsing OPUS for RTC.
-    bool is_sequence_header = is_aac_sequence_header;
+    if ((err = format_->on_audio(msg)) != srs_success) {
+        return srs_error_wrap(err, "format consume audio");
+    }
+
+    // Ignore if no format->acodec, it means the codec is not parsed, or unsupport/unknown codec
+    // such as G.711 codec
+    if (!format_->acodec) {
+        return err;
+    }
+
+    // Whether current packet is sequence header. Note that MP3 does not have one, but we use the first packet as it.
+    bool is_sequence_header = format_->is_aac_sequence_header() || format_->is_mp3_sequence_header();
 
     // whether consumer should drop for the duplicated sequence header.
     bool drop_for_reduce = false;
@@ -2335,10 +2332,8 @@ srs_error_t SrsLiveSource::on_audio_imp(SrsSharedPtrMessage* msg)
         }
     }
     
-    // cache the sequence header of aac, or first packet of mp3.
+    // Refresh the sequence header in metadata.
-    // for example, the mp3 is used for hls to write the "right" audio codec.
+    if (is_sequence_header || !meta->ash()) {
-    // TODO: FIXME: to refine the stream info system.
-    if (is_aac_sequence_header || !meta->ash()) {
         if ((err = meta->update_ash(msg)) != srs_success) {
             return srs_error_wrap(err, "meta consume audio");
         }
@@ -2430,6 +2425,22 @@ srs_error_t SrsLiveSource::on_video_imp(SrsSharedPtrMessage* msg)
 
     bool is_sequence_header = SrsFlvVideo::sh(msg->payload, msg->size);
 
+    // user can disable the sps parse to workaround when parse sps failed.
+    // @see https://github.com/ossrs/srs/issues/474
+    if (is_sequence_header) {
+        format_->avc_parse_sps = _srs_config->get_parse_sps(req->vhost);
+    }
+
+    if ((err = format_->on_video(msg)) != srs_success) {
+        return srs_error_wrap(err, "format consume video");
+    }
+
+    // Ignore if no format->vcodec, it means the codec is not parsed, or unsupport/unknown codec
+    // such as H.263 codec
+    if (!format_->vcodec) {
+        return err;
+    }
+    
     // whether consumer should drop for the duplicated sequence header.
     bool drop_for_reduce = false;
     if (is_sequence_header && meta->previous_vsh() && _srs_config->get_reduce_sequence_header(req->vhost)) {

trunk/src/app/srs_app_source.hpp

@@ -318,8 +318,6 @@ private:
     SrsRequest* req_;
     bool is_active;
 private:
-    // The format, codec information.
-    SrsRtmpFormat* format;
     // hls handler.
     SrsHls* hls;
     // The DASH encoder.
@@ -530,6 +528,8 @@ private:
     SrsOriginHub* hub;
     // The metadata cache.
     SrsMetaCache* meta;
+    // The format, codec information.
+    SrsRtmpFormat* format_;
 private:
     // Whether source is avaiable for publishing.
     bool _can_publish;

trunk/src/core/srs_core_version5.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    116
+#define VERSION_REVISION    117
 
 #endif

trunk/src/kernel/srs_kernel_codec.cpp

@@ -738,6 +738,7 @@ srs_error_t SrsFormat::on_audio(int64_t timestamp, char* data, int size)
         return err;
     }
 
+    bool fresh = !acodec;
     if (!acodec) {
         acodec = new SrsAudioCodecConfig();
     }
@@ -753,7 +754,7 @@ srs_error_t SrsFormat::on_audio(int64_t timestamp, char* data, int size)
     buffer->skip(-1 * buffer->pos());
     
     if (codec == SrsAudioCodecIdMP3) {
-        return audio_mp3_demux(buffer, timestamp);
+        return audio_mp3_demux(buffer, timestamp, fresh);
     }
     
     return audio_aac_demux(buffer, timestamp);
@@ -824,6 +825,12 @@ bool SrsFormat::is_aac_sequence_header()
         && audio && audio->aac_packet_type == SrsAudioAacFrameTraitSequenceHeader;
 }
 
+bool SrsFormat::is_mp3_sequence_header()
+{
+    return acodec && acodec->id == SrsAudioCodecIdMP3
+        && audio && audio->aac_packet_type == SrsAudioMp3FrameTraitSequenceHeader;
+}
+
 bool SrsFormat::is_avc_sequence_header()
 {
     bool h264 = (vcodec && vcodec->id == SrsVideoCodecIdAVC);
@@ -2040,13 +2047,13 @@ srs_error_t SrsFormat::audio_aac_demux(SrsBuffer* stream, int64_t timestamp)
     return err;
 }
 
-srs_error_t SrsFormat::audio_mp3_demux(SrsBuffer* stream, int64_t timestamp)
+srs_error_t SrsFormat::audio_mp3_demux(SrsBuffer* stream, int64_t timestamp, bool fresh)
 {
     srs_error_t err = srs_success;
     
     audio->cts = 0;
     audio->dts = timestamp;
-    audio->aac_packet_type = SrsAudioMp3FrameTrait;
+    audio->aac_packet_type = fresh ? SrsAudioMp3FrameTraitSequenceHeader : SrsAudioMp3FrameTraitRawData;
     
     // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76
     int8_t sound_format = stream->read_1bytes();

trunk/src/kernel/srs_kernel_codec.hpp

@@ -162,6 +162,7 @@ enum SrsAudioAacFrameTrait
     SrsAudioAacFrameTraitReserved = 0xff,
     SrsAudioAacFrameTraitForbidden = 0xff,
 
+    // For AAC, we detect the sequence header by content.
     SrsAudioAacFrameTraitSequenceHeader = 0,
     SrsAudioAacFrameTraitRawData = 1,
     
@@ -172,8 +173,10 @@ enum SrsAudioAacFrameTrait
 
     // 16/32 reserved for g711a/g711u 
 
-    // For MP3
+    // For MP3 we assume the first packet is sequence header, while it actually is not the same thing, because we do
-    SrsAudioMp3FrameTrait = 64,
+    // this to simplify the workflow, to make sure we can detect the audio codec from the sequence headers.
+    SrsAudioMp3FrameTraitSequenceHeader = 63,
+    SrsAudioMp3FrameTraitRawData = 64,
 };
 
 /**
@@ -972,6 +975,7 @@ public:
     virtual srs_error_t on_aac_sequence_header(char* data, int size);
 public:
     virtual bool is_aac_sequence_header();
+    virtual bool is_mp3_sequence_header();
     virtual bool is_avc_sequence_header();
 private:
     // Demux the video packet in H.264 codec.
@@ -1006,7 +1010,7 @@ private:
     //          Demux the asc from sequence header.
     //          Demux the sampels from RAW data.
     virtual srs_error_t audio_aac_demux(SrsBuffer* stream, int64_t timestamp);
-    virtual srs_error_t audio_mp3_demux(SrsBuffer* stream, int64_t timestamp);
+    virtual srs_error_t audio_mp3_demux(SrsBuffer* stream, int64_t timestamp, bool fresh);
 public:
     // Directly demux the sequence header, without RTMP packet header.
     virtual srs_error_t audio_aac_sequence_header_demux(char* data, int size);

trunk/src/utest/srs_utest_kernel.cpp

@@ -5734,7 +5734,7 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleAVsWithMp3)
 
         // Frames order by dts asc.
         HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
-        EXPECT_EQ(0, (int)dts); EXPECT_EQ(127, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeVIDE, ht); EXPECT_NE(SrsAudioMp3FrameTrait, ct);
+        EXPECT_EQ(0, (int)dts); EXPECT_EQ(127, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeVIDE, ht); EXPECT_NE(SrsAudioMp3FrameTraitSequenceHeader, ct);
         srs_freepa(sample);
 
         HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
@@ -5746,7 +5746,7 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleAVsWithMp3)
         srs_freepa(sample);
 
         HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
-        EXPECT_EQ(40, (int)dts); EXPECT_EQ(40, (int)pts); EXPECT_EQ(127, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeVIDE, ht); EXPECT_NE(SrsAudioMp3FrameTrait, ct);
+        EXPECT_EQ(40, (int)dts); EXPECT_EQ(40, (int)pts); EXPECT_EQ(127, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeVIDE, ht); EXPECT_NE(SrsAudioMp3FrameTraitSequenceHeader, ct);
         srs_freepa(sample);
     }
 }