From 3683f06e4de36b09a603f65f1f913b3808414846 Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 27 Oct 2015 16:20:02 +0800 Subject: [PATCH 1/2] for #512, write audio frame by frame for video+audio hls. --- README.md | 6 ++++- trunk/src/app/srs_app_hls.cpp | 37 ++++++++++----------------- trunk/src/app/srs_app_hls.hpp | 4 +++ trunk/src/kernel/srs_kernel_codec.hpp | 6 ----- trunk/src/kernel/srs_kernel_ts.cpp | 22 ++++++---------- trunk/src/kernel/srs_kernel_ts.hpp | 5 ++++ 6 files changed, 36 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 3b27b2f10..57a753539 100755 --- a/README.md +++ b/README.md @@ -765,7 +765,9 @@ About the HLS overhead of SRS, we compare the overhead to FLV by remux the HLS t | 5147kbps | 370s | 195040 | 200280 | 2.68% | | 5158kbps | 1327s | 835664 | 858092 | 2.68% | -The HLS overhead is calc by: (HLS - FLV) / FLV * 100% +The HLS overhead is calc by: (HLS - FLV) / FLV * 100%. + +The overhead is larger than this benchmark(48kbps audio is best overhead), for we fix the [#512][bug#512]. ## Architecture @@ -1193,6 +1195,8 @@ Winlin [bug #59]: https://github.com/simple-rtmp-server/srs/issues/59 [bug #50]: https://github.com/simple-rtmp-server/srs/issues/50 [bug #34]: https://github.com/simple-rtmp-server/srs/issues/34 +[bug #512]: https://github.com/simple-rtmp-server/srs/issues/512 +[bug #xxxxxxxxxx]: https://github.com/simple-rtmp-server/srs/issues/xxxxxxxxxx [r2.0a2]: https://github.com/simple-rtmp-server/srs/releases/tag/v2.0-a2 [r2.0a1]: https://github.com/simple-rtmp-server/srs/releases/tag/2.0a1 diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 8b36ab7f9..64a8396c0 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -646,6 +646,11 @@ int SrsHlsMuxer::update_acodec(SrsCodecAudio ac) return current->muxer->update_acodec(ac); } +bool SrsHlsMuxer::pure_audio() +{ + return current && current->muxer && current->muxer->video_codec() == SrsCodecVideoDisabled; +} + int SrsHlsMuxer::flush_audio(SrsTsCache* cache) { int ret = ERROR_SUCCESS; @@ -1049,25 +1054,6 @@ int SrsHlsCache::write_audio(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t return ret; } - // flush if buffer exceed max size. - if (cache->audio->payload->length() > SRS_AUTO_HLS_AUDIO_CACHE_SIZE) { - if ((ret = muxer->flush_audio(cache)) != ERROR_SUCCESS) { - return ret; - } - } - - // TODO: config it. - // in ms, audio delay to flush the audios. - int64_t audio_delay = SRS_CONF_DEFAULT_AAC_DELAY; - // flush if audio delay exceed - // cache->audio will be free in flush_audio - // so we must check whether it's null ptr. - if (cache->audio && pts - cache->audio->start_pts > audio_delay * 90) { - if ((ret = muxer->flush_audio(cache)) != ERROR_SUCCESS) { - return ret; - } - } - // reap when current source is pure audio. // it maybe changed when stream info changed, // for example, pure audio when start, audio/video when publishing, @@ -1083,6 +1069,14 @@ int SrsHlsCache::write_audio(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t } } + // directly write the audio frame by frame to ts, + // it's ok for the hls overload, or maybe cause the audio corrupt, + // which introduced by aggregate the audios to a big one. + // @see https://github.com/simple-rtmp-server/srs/issues/512 + if ((ret = muxer->flush_audio(cache)) != ERROR_SUCCESS) { + return ret; + } + return ret; } @@ -1100,7 +1094,7 @@ int SrsHlsCache::write_video(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t // do reap ts if any of: // a. wait keyframe and got keyframe. // b. always reap when not wait keyframe. - if (!muxer->wait_keyframe()|| sample->frame_type == SrsCodecVideoAVCFrameKeyFrame) { + if (!muxer->wait_keyframe() || sample->frame_type == SrsCodecVideoAVCFrameKeyFrame) { // when wait keyframe, there must exists idr frame in sample. if (!sample->has_idr && muxer->wait_keyframe()) { srs_warn("hls: ts starts without IDR, first nalu=%d, idr=%d", sample->first_nalu_type, sample->has_idr); @@ -1110,9 +1104,6 @@ int SrsHlsCache::write_video(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t if ((ret = reap_segment("video", muxer, cache->video->dts)) != ERROR_SUCCESS) { return ret; } - - // the video must be flushed, just return. - return ret; } } diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp index 62abd660f..a9269d69e 100644 --- a/trunk/src/app/srs_app_hls.hpp +++ b/trunk/src/app/srs_app_hls.hpp @@ -309,6 +309,10 @@ public: virtual bool is_segment_absolutely_overflow(); public: virtual int update_acodec(SrsCodecAudio ac); + /** + * whether current hls muxer is pure audio mode. + */ + virtual bool pure_audio(); virtual int flush_audio(SrsTsCache* cache); virtual int flush_video(SrsTsCache* cache); /** diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index c47628276..cc995dd04 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -246,12 +246,6 @@ extern int aac_sample_rates[]; #define SRS_SRS_MAX_CODEC_SAMPLE 128 #define SRS_AAC_SAMPLE_RATE_UNSET 15 -// in ms, for HLS aac flush the audio -#define SRS_CONF_DEFAULT_AAC_DELAY 60 - -// max PES packets size to flush the video. -#define SRS_AUTO_HLS_AUDIO_CACHE_SIZE 128 * 1024 - /** * the FLV/RTMP supported audio sample size. * Size of each audio sample. This parameter only pertains to diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 0ae7d58a5..864e05aa8 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -2760,6 +2760,11 @@ void SrsTSMuxer::close() writer->close(); } +SrsCodecVideo SrsTSMuxer::video_codec() +{ + return vcodec; +} + SrsTsCache::SrsTsCache() { audio = NULL; @@ -3134,20 +3139,9 @@ int SrsTsEncoder::write_audio(int64_t timestamp, char* data, int size) return ret; } - // flush if buffer exceed max size. - if (cache->audio->payload->length() > SRS_AUTO_HLS_AUDIO_CACHE_SIZE) { - return flush_video(); - } - - // TODO: config it. - // in ms, audio delay to flush the audios. - int64_t audio_delay = SRS_CONF_DEFAULT_AAC_DELAY; - // flush if audio delay exceed - if (dts - cache->audio->start_pts > audio_delay * 90) { - return flush_audio(); - } - - return ret; + // always flush audio frame by frame. + // @see https://github.com/simple-rtmp-server/srs/issues/512 + return flush_audio(); } int SrsTsEncoder::write_video(int64_t timestamp, char* data, int size) diff --git a/trunk/src/kernel/srs_kernel_ts.hpp b/trunk/src/kernel/srs_kernel_ts.hpp index e20778afc..3003f6cf3 100644 --- a/trunk/src/kernel/srs_kernel_ts.hpp +++ b/trunk/src/kernel/srs_kernel_ts.hpp @@ -1586,6 +1586,11 @@ public: * close the writer. */ virtual void close(); +public: + /** + * get the video codec of ts muxer. + */ + virtual SrsCodecVideo video_codec(); }; /** From d1979c760f6d3e5a13644e72e999242495ea64a0 Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 27 Oct 2015 17:44:10 +0800 Subject: [PATCH 2/2] for #512, partical hotfix the hls pure audio. 2.0.196 --- README.md | 1 + trunk/src/app/srs_app_hls.cpp | 10 +++++++++- trunk/src/core/srs_core.hpp | 2 +- trunk/src/kernel/srs_kernel_ts.cpp | 14 ++++++++++---- trunk/src/kernel/srs_kernel_ts.hpp | 4 ++++ 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 57a753539..e6e18c489 100755 --- a/README.md +++ b/README.md @@ -336,6 +336,7 @@ Remark: ## History +* v2.0, 2015-10-27, for [#512][bug #512] partical hotfix the hls pure audio. 2.0.196 * v2.0, 2015-10-08, [2.0 alpha2(2.0.195)][r2.0a2] released. 89358 lines. * v2.0, 2015-10-04, for [#448][bug #448] fix the bug of response of http hooks. 2.0.195 * v2.0, 2015-10-01, for [#497][bug #497] response error when client not found to kickoff. 2.0.194 diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 64a8396c0..71360f6ef 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -578,6 +578,7 @@ int SrsHlsMuxer::segment_open(int64_t segment_start_dts) current->full_path.c_str(), tmp_file.c_str()); // set the segment muxer audio codec. + // TODO: FIXME: refine code, use event instead. if (acodec != SrsCodecAudioReserved1) { current->muxer->update_acodec(acodec); } @@ -1044,7 +1045,7 @@ int SrsHlsCache::on_sequence_header(SrsHlsMuxer* muxer) // when the sequence header changed, the stream is not republish. return muxer->on_sequence_header(); } - + int SrsHlsCache::write_audio(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t pts, SrsCodecSample* sample) { int ret = ERROR_SUCCESS; @@ -1069,6 +1070,13 @@ int SrsHlsCache::write_audio(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t } } + // for pure audio, aggregate some frame to one. + if (muxer->pure_audio() && cache->audio) { + if (pts - cache->audio->start_pts < SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE) { + return ret; + } + } + // directly write the audio frame by frame to ts, // it's ok for the hls overload, or maybe cause the audio corrupt, // which introduced by aggregate the audios to a big one. diff --git a/trunk/src/core/srs_core.hpp b/trunk/src/core/srs_core.hpp index d4d96eb3e..d059c1afa 100644 --- a/trunk/src/core/srs_core.hpp +++ b/trunk/src/core/srs_core.hpp @@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // current release version #define VERSION_MAJOR 2 #define VERSION_MINOR 0 -#define VERSION_REVISION 195 +#define VERSION_REVISION 196 // server info. #define RTMP_SIG_SRS_KEY "SRS" diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 864e05aa8..e0a9961a6 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -459,8 +459,11 @@ int SrsTsContext::encode_pes(SrsFileWriter* writer, SrsTsMessage* msg, int16_t p while (p < end) { SrsTsPacket* pkt = NULL; if (p == start) { - // for pure audio stream, always write pcr. + // write pcr according to message. bool write_pcr = msg->write_pcr; + + // for pure audio, always write pcr. + // TODO: FIXME: maybe only need to write at begin and end of ts. if (pure_audio && msg->is_audio()) { write_pcr = true; } @@ -2785,11 +2788,12 @@ int SrsTsCache::cache_audio(SrsAvcAacCodec* codec, int64_t dts, SrsCodecSample* if (!audio) { audio = new SrsTsMessage(); audio->write_pcr = false; - audio->start_pts = dts; + audio->dts = audio->pts = audio->start_pts = dts; } - audio->dts = dts; - audio->pts = audio->dts; + // TODO: FIXME: refine code. + //audio->dts = dts; + //audio->pts = audio->dts; audio->sid = SrsTsPESStreamIdAudioCommon; // must be aac or mp3 @@ -3139,6 +3143,8 @@ int SrsTsEncoder::write_audio(int64_t timestamp, char* data, int size) return ret; } + // TODO: FIXME: for pure audio, aggregate some frame to one. + // always flush audio frame by frame. // @see https://github.com/simple-rtmp-server/srs/issues/512 return flush_audio(); diff --git a/trunk/src/kernel/srs_kernel_ts.hpp b/trunk/src/kernel/srs_kernel_ts.hpp index 3003f6cf3..42d40a8af 100644 --- a/trunk/src/kernel/srs_kernel_ts.hpp +++ b/trunk/src/kernel/srs_kernel_ts.hpp @@ -54,6 +54,9 @@ class SrsTsContext; // Transport Stream packets are 188 bytes in length. #define SRS_TS_PACKET_SIZE 188 +// the aggregate pure audio for hls, in ts tbn(ms * 90). +#define SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE 720 * 90 + /** * the pid of ts packet, * Table 2-3 - PID table, hls-mpeg-ts-iso13818-1.pdf, page 37 @@ -359,6 +362,7 @@ public: /** * whether the hls stream is pure audio stream. */ + // TODO: FIXME: merge with muxer codec detect. virtual bool is_pure_audio(); /** * when PMT table parsed, we know some info about stream.