RTC: Fix FFmpeg opus audio noisy issue. v5.0.195 v6.0.95 (#3845)

Follow the example in FFmpeg's doc, before calling the API
`avcodec_send_frame`, always use `av_frame_alloc` to create a new frame.

---------

Co-authored-by: Haibo Chen <495810242@qq.com>

trunk/auto/options.sh

@@ -37,7 +37,7 @@ SRS_FFMPEG_TOOL=NO
 # FFmpeg fit is the source code for RTC, to transcode audio or video in SRS.
 SRS_FFMPEG_FIT=RESERVED
 # Whether use FFmpeg native opus codec for RTC. If not, use libopus instead.
-SRS_FFMPEG_OPUS=NO
+SRS_FFMPEG_OPUS=YES
 # arguments
 SRS_PREFIX=/usr/local/srs
 SRS_DEFAULT_CONFIG=conf/srs.conf

trunk/conf/rtc.tcp.only.conf

@@ -0,0 +1,46 @@
+# WebRTC streaming config for SRS.
+# @see full.conf for detail config.
+
+listen              1935;
+max_connections     1000;
+daemon              off;
+srs_log_tank        console;
+
+http_server {
+    enabled         on;
+    listen          8080;
+    dir             ./objs/nginx/html;
+}
+
+http_api {
+    enabled         on;
+    listen          1985;
+}
+stats {
+    network         0;
+}
+rtc_server {
+    enabled on;
+    tcp {
+        enabled on;
+        listen 8000;
+    }
+    protocol tcp;
+    # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#config-candidate
+    candidate $CANDIDATE;
+}
+
+vhost __defaultVhost__ {
+    rtc {
+        enabled     on;
+        # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtmp-to-rtc
+        rtmp_to_rtc off;
+        # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtc-to-rtmp
+        rtc_to_rtmp off;
+    }
+    http_remux {
+        enabled     on;
+        mount       [vhost]/[app]/[stream].flv;
+    }
+}
+

trunk/conf/rtc.tcp.udp.conf

@@ -0,0 +1,47 @@
+# WebRTC streaming config for SRS.
+# @see full.conf for detail config.
+
+listen              1935;
+max_connections     1000;
+daemon              off;
+srs_log_tank        console;
+
+http_server {
+    enabled         on;
+    listen          8080;
+    dir             ./objs/nginx/html;
+}
+
+http_api {
+    enabled         on;
+    listen          1985;
+}
+stats {
+    network         0;
+}
+rtc_server {
+    enabled on;
+    listen 8000; # UDP port
+    tcp {
+        enabled on;
+        listen 8000;
+    }
+    protocol all;
+    # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#config-candidate
+    candidate $CANDIDATE;
+}
+
+vhost __defaultVhost__ {
+    rtc {
+        enabled     on;
+        # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtmp-to-rtc
+        rtmp_to_rtc off;
+        # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/webrtc#rtc-to-rtmp
+        rtc_to_rtmp off;
+    }
+    http_remux {
+        enabled     on;
+        mount       [vhost]/[app]/[stream].flv;
+    }
+}
+

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v6-changes"></a>
 
 ## SRS 6.0 Changelog
+* v6.0, 2023-10-25, Merge [#3845](https://github.com/ossrs/srs/pull/3845): RTC: Fix FFmpeg opus audio noisy issue. v6.0.95 (#3845)
 * v6.0, 2023-10-21, Merge [#3847](https://github.com/ossrs/srs/pull/3847): WebRTC: TCP transport should use read_fully instead of read. v6.0.94 (#3847)
 * v6.0, 2023-10-20, Merge [#3846](https://github.com/ossrs/srs/pull/3846): Added system library option for ffmpeg, srtp, srt libraries. v6.0.93 (#3846)
 * v6.0, 2023-10-17, Merge [#3840](https://github.com/ossrs/srs/pull/3840): Disable asan by default. v6.0.92 (#3840)
@@ -106,6 +107,7 @@ The changelog for SRS.
 <a name="v5-changes"></a>
 
 ## SRS 5.0 Changelog
+* v5.0, 2023-10-25, Merge [#3845](https://github.com/ossrs/srs/pull/3845): RTC: Fix FFmpeg opus audio noisy issue. v5.0.195 (#3845)
 * v5.0, 2023-10-21, Merge [#3847](https://github.com/ossrs/srs/pull/3847): WebRTC: TCP transport should use read_fully instead of read. v5.0.194 (#3847)
 * v5.0, 2023-10-20, Merge [#3846](https://github.com/ossrs/srs/pull/3846): Added system library option for ffmpeg, srtp, srt libraries. v5.0.193 (#3846)
 * v5.0, 2023-10-17, Merge [#3840](https://github.com/ossrs/srs/pull/3840): Disable asan by default. v5.0.192 (#3840)

trunk/ide/srs_clion/CMakeLists.txt

@@ -44,7 +44,6 @@ set(DEPS_LIBS ${SRS_DIR}/objs/st/libst.a
         ${SRS_DIR}/objs/srtp2/lib/libsrtp2.a
         ${SRS_DIR}/objs/ffmpeg/lib/libavcodec.a
         ${SRS_DIR}/objs/ffmpeg/lib/libavutil.a
-        ${SRS_DIR}/objs/opus/lib/libopus.a
         ${SRS_DIR}/objs/ffmpeg/lib/libswresample.a
         ${SRS_DIR}/objs/srt/lib/libsrt.a)
 foreach(DEPS_LIB ${DEPS_LIBS})

trunk/src/app/srs_app_rtc_codec.cpp

@@ -242,7 +242,7 @@ srs_error_t SrsAudioTranscoder::init_enc(SrsAudioCodecId dst_codec, int dst_chan
     enc_->channel_layout = av_get_default_channel_layout(dst_channels);
     enc_->bit_rate = dst_bit_rate;
     enc_->sample_fmt = codec->sample_fmts[0];
-    enc_->time_base.num = 1; enc_->time_base.den = 1000; // {1, 1000}
+    enc_->time_base.num = 1; enc_->time_base.den = dst_samplerate; // {1, dst_samplerate}
     if (dst_codec == SrsAudioCodecIdOpus) {
         //TODO: for more level setting
         enc_->compression_level = 1;
@@ -261,14 +261,6 @@ srs_error_t SrsAudioTranscoder::init_enc(SrsAudioCodecId dst_codec, int dst_chan
         return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not allocate audio encode in frame");
     }
 
-    enc_frame_->format = enc_->sample_fmt;
-    enc_frame_->nb_samples = enc_->frame_size;
-    enc_frame_->channel_layout = enc_->channel_layout;
-
-    if (av_frame_get_buffer(enc_frame_, 0) < 0) {
-        return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not get audio frame buffer");
-    }
-
     enc_packet_ = av_packet_alloc();
     if (!enc_packet_) {
         return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not allocate audio encode out packet");
@@ -380,25 +372,35 @@ srs_error_t SrsAudioTranscoder::encode(std::vector<SrsAudioFrame*> &pkts)
     if (next_out_pts_ == AV_NOPTS_VALUE) {
         next_out_pts_ = new_pkt_pts_;
     } else {
-        int64_t diff = llabs(new_pkt_pts_ - next_out_pts_);
+        int64_t diff = llabs(new_pkt_pts_ - av_rescale(next_out_pts_, 1000, enc_->time_base.den));
         if (diff > 1000) {
             srs_trace("time diff to large=%lld, next out=%lld, new pkt=%lld, set to new pkt",
                 diff, next_out_pts_, new_pkt_pts_);
-            next_out_pts_ = new_pkt_pts_;
+            next_out_pts_ = av_rescale(new_pkt_pts_, enc_->time_base.den, 1000);
         }
     }
 
-    int frame_cnt = 0;
     while (av_audio_fifo_size(fifo_) >= enc_->frame_size) {
+        enc_frame_->format = enc_->sample_fmt;
+        enc_frame_->nb_samples = enc_->frame_size;
+        enc_frame_->channel_layout = enc_->channel_layout;
+
+        if (av_frame_get_buffer(enc_frame_, 0) < 0) {
+            av_frame_free(&enc_frame_);
+            return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not get audio frame buffer");
+        }
+
         /* Read as many samples from the FIFO buffer as required to fill the frame.
         * The samples are stored in the frame temporarily. */
         if (av_audio_fifo_read(fifo_, (void **)enc_frame_->data, enc_->frame_size) < enc_->frame_size) {
+            av_frame_free(&enc_frame_);
             return srs_error_new(ERROR_RTC_RTP_MUXER, "Could not read data from FIFO");
         }
         /* send the frame for encoding */
-        enc_frame_->pts = next_out_pts_ + av_rescale(enc_->frame_size * frame_cnt, 1000, enc_->sample_rate);
-        ++frame_cnt;
+        enc_frame_->pts = next_out_pts_;
+        next_out_pts_ += enc_->frame_size;
         int error = avcodec_send_frame(enc_, enc_frame_);
+        av_frame_unref(enc_frame_);
         if (error < 0) {
             return srs_error_new(ERROR_RTC_RTP_MUXER, "Error sending the frame to the encoder(%d,%s)", error,
                 av_make_error_string(err_buf, AV_ERROR_MAX_STRING_SIZE, error));
@@ -419,6 +421,10 @@ srs_error_t SrsAudioTranscoder::encode(std::vector<SrsAudioFrame*> &pkts)
                     av_make_error_string(err_buf, AV_ERROR_MAX_STRING_SIZE, error));
             }
 
+            // rescale time base from sample_rate 1000.
+            enc_packet_->dts = av_rescale(enc_packet_->dts, 1000, enc_->time_base.den); 
+            enc_packet_->pts = av_rescale(enc_packet_->pts, 1000, enc_->time_base.den);
+
             SrsAudioFrame *out_frame = new SrsAudioFrame;
             char *buf = new char[enc_packet_->size];
             memcpy(buf, enc_packet_->data, enc_packet_->size);
@@ -429,8 +435,6 @@ srs_error_t SrsAudioTranscoder::encode(std::vector<SrsAudioFrame*> &pkts)
         }
     }
 
-    next_out_pts_ += av_rescale(enc_->frame_size * frame_cnt, 1000, enc_->sample_rate);
-
     return srs_success;
 }
 

trunk/src/app/srs_app_source.cpp

@@ -2721,7 +2721,7 @@ srs_error_t SrsLiveSource::consumer_dumps(SrsLiveConsumer* consumer, bool ds, bo
 
     // print status.
     if (dg) {
-        srs_trace("create consumer, active=%d, queue_size=%.2f, jitter=%d", hub->active(), queue_size, jitter_algorithm);
+        srs_trace("create consumer, active=%d, queue_size=%dms, jitter=%d", hub->active(), srsu2msi(queue_size), jitter_algorithm);
     } else {
         srs_trace("create consumer, active=%d, ignore gop cache, jitter=%d", hub->active(), jitter_algorithm);
     }

trunk/src/core/srs_core_version5.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    194
+#define VERSION_REVISION    195
 
 #endif

trunk/src/core/srs_core_version6.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       6
 #define VERSION_MINOR       0
-#define VERSION_REVISION    94
+#define VERSION_REVISION    95
 
 #endif

trunk/src/protocol/srs_protocol_rtmp_handshake.cpp

@@ -298,8 +298,6 @@ namespace srs_internal
     {
         srs_error_t err = srs_success;
         
-        int32_t bits_count = 1024;
-        
         close();
         
         //1. Create the DH