From f8855cfcbb7dbe9bb59973d33759707dccec7842 Mon Sep 17 00:00:00 2001 From: winlin Date: Wed, 27 Nov 2013 17:30:16 +0800 Subject: [PATCH] fix the aac jump bug on iphone, correct the audio pts, use audio buffer and flush audio --- README.md | 2 +- trunk/conf/srs.conf | 4 +- trunk/src/core/srs_core_codec.cpp | 21 ++- trunk/src/core/srs_core_config.hpp | 4 + trunk/src/core/srs_core_hls.cpp | 266 ++++++++++++++++++++++------- trunk/src/core/srs_core_hls.hpp | 50 +++++- trunk/src/core/srs_core_source.cpp | 2 +- trunk/src/core/srs_core_source.hpp | 4 +- 8 files changed, 281 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 1c62f11ec..e8e9a4598 100755 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ For example, use ffmpeg to publish: step 5: play live stream 
 rtmp url: rtmp://127.0.0.1:1935/live/livestream
-m3u8 url: http://127.0.0.1:1935/live/livestream.m3u8
+m3u8 url: http://127.0.0.1:80/live/livestream.m3u8
### Summary diff --git a/trunk/conf/srs.conf b/trunk/conf/srs.conf index cde87f82f..8ff97257a 100755 --- a/trunk/conf/srs.conf +++ b/trunk/conf/srs.conf @@ -13,8 +13,8 @@ vhost __defaultVhost__ { gop_cache on; hls on; hls_path ./objs/nginx/html; - hls_fragment 10; - hls_window 60; + hls_fragment 5; + hls_window 30; } # the vhost disabled. vhost removed.vhost.com { diff --git a/trunk/src/core/srs_core_codec.cpp b/trunk/src/core/srs_core_codec.cpp index 88b270092..24faa9376 100644 --- a/trunk/src/core/srs_core_codec.cpp +++ b/trunk/src/core/srs_core_codec.cpp @@ -159,7 +159,7 @@ int SrsCodec::audio_aac_demux(int8_t* data, int size, SrsCodecSample* sample) int8_t sound_type = sound_format & 0x01; int8_t sound_size = (sound_format >> 1) & 0x01; - int8_t sound_rate = (sound_format >> 2) & 0x01; + int8_t sound_rate = (sound_format >> 2) & 0x03; sound_format = (sound_format >> 4) & 0x0f; audio_codec_id = sound_format; @@ -167,6 +167,25 @@ int SrsCodec::audio_aac_demux(int8_t* data, int size, SrsCodecSample* sample) sample->sound_rate = (SrsCodecAudioSampleRate)sound_rate; sample->sound_size = (SrsCodecAudioSampleSize)sound_size; + // reset the sample rate by sequence header + static int aac_sample_rates[] = { + 96000, 88200, 64000, 48000, + 44100, 32000, 24000, 22050, + 16000, 12000, 11025, 8000, + 7350, 0, 0, 0 + }; + switch (aac_sample_rates[aac_sample_rate]) { + case 11025: + sample->sound_rate = SrsCodecAudioSampleRate11025; + break; + case 22050: + sample->sound_rate = SrsCodecAudioSampleRate22050; + break; + case 44100: + sample->sound_rate = SrsCodecAudioSampleRate44100; + break; + }; + // only support aac if (audio_codec_id != SrsCodecAudioAAC) { ret = ERROR_HLS_DECODE_ERROR; diff --git a/trunk/src/core/srs_core_config.hpp b/trunk/src/core/srs_core_config.hpp index 2c0adeb88..b02c1efbd 100644 --- a/trunk/src/core/srs_core_config.hpp +++ b/trunk/src/core/srs_core_config.hpp @@ -40,6 +40,10 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define SRS_CONF_DEFAULT_HLS_PATH "./objs/nginx/html" #define SRS_CONF_DEFAULT_HLS_FRAGMENT 10 #define SRS_CONF_DEFAULT_HLS_WINDOW 60 +// in ms, for HLS aac sync time. +#define SRS_CONF_DEFAULT_AAC_SYNC 100 +// in ms, for HLS aac flush the audio +#define SRS_CONF_DEFAULT_AAC_DELAY 300 class SrsFileBuffer { diff --git a/trunk/src/core/srs_core_hls.cpp b/trunk/src/core/srs_core_hls.cpp index 8882d5e3e..a8335aef2 100644 --- a/trunk/src/core/srs_core_hls.cpp +++ b/trunk/src/core/srs_core_hls.cpp @@ -269,9 +269,6 @@ public: } } - // write success, clear and free the buffer - buffer->free(); - return ret; } private: @@ -360,16 +357,79 @@ SrsM3u8Segment::~SrsM3u8Segment() srs_freep(muxer); } -SrsHLS::SrsHLS() +SrsHlsAacJitter::SrsHlsAacJitter() +{ + base_pts = 0; + nb_samples = 0; + + // TODO: config it, 0 means no adjust + sync_ms = SRS_CONF_DEFAULT_AAC_SYNC; +} + +SrsHlsAacJitter::~SrsHlsAacJitter() +{ +} + +int64_t SrsHlsAacJitter::on_buffer_start(int64_t flv_pts, int sample_rate) +{ + // 0 = 5.5 kHz = 5512 Hz + // 1 = 11 kHz = 11025 Hz + // 2 = 22 kHz = 22050 Hz + // 3 = 44 kHz = 44100 Hz + static int flv_sample_rates[] = {5512, 11025, 22050, 44100}; + int flv_sample_rate = flv_sample_rates[sample_rate & 0x03]; + + // sync time set to 0, donot adjust the aac timestamp. + if (!sync_ms) { + return flv_pts; + } + + // @see: ngx_rtmp_hls_audio + /* TODO: We assume here AAC frame size is 1024 + * Need to handle AAC frames with frame size of 960 */ + int64_t est_pts = base_pts + nb_samples * 90000LL * 1024LL / flv_sample_rate; + int64_t dpts = (int64_t) (est_pts - flv_pts); + + if (dpts <= (int64_t) sync_ms * 90 && dpts >= (int64_t) sync_ms * -90) { + srs_info("HLS correct aac pts " + "from %"PRId64" to %"PRId64", base=%"PRId64", nb_samples=%d, sample_rate=%d", + flv_pts, est_pts, nb_samples, flv_sample_rate, base_pts); + + nb_samples++; + + return est_pts; + } + + // resync + srs_trace("HLS aac resync, dpts=%"PRId64", pts=%"PRId64 + ", base=%"PRId64", nb_samples=%"PRId64", sample_rate=%d", + dpts, flv_pts, base_pts, nb_samples, flv_sample_rate); + + base_pts = flv_pts; + nb_samples = 1; + + return flv_pts; +} + +void SrsHlsAacJitter::on_buffer_continue() +{ + nb_samples++; +} + +SrsHls::SrsHls() { hls_enabled = false; codec = new SrsCodec(); sample = new SrsCodecSample(); current = NULL; jitter = new SrsRtmpJitter(); + aac_jitter = new SrsHlsAacJitter(); file_index = 0; - m3u8_dts = stream_dts = 0; + audio_buffer_start_pts = m3u8_dts = stream_dts = 0; hls_fragment = hls_window = 0; + + // TODO: config it. + audio_delay = SRS_CONF_DEFAULT_AAC_DELAY; audio_buffer = new SrsCodecBuffer(); video_buffer = new SrsCodecBuffer(); @@ -378,11 +438,12 @@ SrsHLS::SrsHLS() video_frame = new SrsMpegtsFrame(); } -SrsHLS::~SrsHLS() +SrsHls::~SrsHls() { srs_freep(codec); srs_freep(sample); srs_freep(jitter); + srs_freep(aac_jitter); std::vector::iterator it; for (it = segments.begin(); it != segments.end(); ++it) { @@ -403,7 +464,7 @@ SrsHLS::~SrsHLS() srs_freep(video_frame); } -int SrsHLS::on_publish(std::string _vhost, std::string _app, std::string _stream) +int SrsHls::on_publish(std::string _vhost, std::string _app, std::string _stream) { int ret = ERROR_SUCCESS; @@ -435,12 +496,12 @@ int SrsHLS::on_publish(std::string _vhost, std::string _app, std::string _stream return ret; } -void SrsHLS::on_unpublish() +void SrsHls::on_unpublish() { hls_enabled = false; } -int SrsHLS::on_meta_data(SrsOnMetaDataPacket* metadata) +int SrsHls::on_meta_data(SrsOnMetaDataPacket* metadata) { int ret = ERROR_SUCCESS; @@ -492,7 +553,7 @@ int SrsHLS::on_meta_data(SrsOnMetaDataPacket* metadata) return ret; } -int SrsHLS::on_audio(SrsSharedPtrMessage* audio) +int SrsHls::on_audio(SrsSharedPtrMessage* audio) { int ret = ERROR_SUCCESS; @@ -523,18 +584,43 @@ int SrsHLS::on_audio(SrsSharedPtrMessage* audio) srs_assert(current); - stream_dts = audio_frame->dts = audio_frame->pts = audio->header.timestamp * 90; - audio_frame->pid = TS_AUDIO_PID; - audio_frame->sid = TS_AUDIO_AAC; + // the pts calc from rtmp/flv header. + int64_t pts = audio->header.timestamp * 90; - if ((ret = current->muxer->write_audio(audio_frame, audio_buffer, codec, sample)) != ERROR_SUCCESS) { + // flush if audio delay exceed + if (pts - audio_buffer_start_pts > audio_delay * 90) { + if ((ret = flush_audio()) != ERROR_SUCCESS) { + return ret; + } + } + + // start buffer, set the audio_frame + if (audio_buffer->size == 0) { + pts = aac_jitter->on_buffer_start(pts, sample->sound_rate); + + audio_frame->dts = audio_frame->pts = audio_buffer_start_pts = pts; + audio_frame->pid = TS_AUDIO_PID; + audio_frame->sid = TS_AUDIO_AAC; + } else { + aac_jitter->on_buffer_continue(); + } + + // write audio to cache. + if ((ret = write_audio()) != ERROR_SUCCESS) { return ret; } + // write cache to file. + if (audio_buffer->size > 1024 * 1024) { + if ((ret = flush_audio()) != ERROR_SUCCESS) { + return ret; + } + } + return ret; } -int SrsHLS::on_video(SrsSharedPtrMessage* video) +int SrsHls::on_video(SrsSharedPtrMessage* video) { int ret = ERROR_SUCCESS; @@ -563,6 +649,11 @@ int SrsHLS::on_video(SrsSharedPtrMessage* video) return ret; } + // write video to cache. + if ((ret = write_video()) != ERROR_SUCCESS) { + return ret; + } + stream_dts = video_frame->dts = video->header.timestamp * 90; video_frame->pts = video_frame->dts + sample->cts * 90; video_frame->pid = TS_VIDEO_PID; @@ -580,14 +671,17 @@ int SrsHLS::on_video(SrsSharedPtrMessage* video) } srs_assert(current); - if ((ret = current->muxer->write_video(video_frame, video_buffer, codec, sample)) != ERROR_SUCCESS) { + if ((ret = current->muxer->write_video(video_frame, video_buffer)) != ERROR_SUCCESS) { return ret; } + + // write success, clear and free the buffer + video_buffer->free(); return ret; } -int SrsHLS::reopen() +int SrsHls::reopen() { int ret = ERROR_SUCCESS; @@ -675,10 +769,19 @@ int SrsHLS::reopen() } srs_info("open HLS muxer success. vhost=%s, path=%s", vhost.c_str(), current->full_path.c_str()); + // segment open, flush the audio. + // @see: ngx_rtmp_hls_open_fragment + /* start fragment with audio to make iPhone happy */ + if (current->muxer->fresh()) { + if ((ret = flush_audio()) != ERROR_SUCCESS) { + return ret; + } + } + return ret; } -int SrsHLS::refresh_m3u8() +int SrsHls::refresh_m3u8() { int ret = ERROR_SUCCESS; @@ -708,7 +811,7 @@ int SrsHLS::refresh_m3u8() return ret; } -int SrsHLS::_refresh_m3u8(int& fd, std::string m3u8_file) +int SrsHls::_refresh_m3u8(int& fd, std::string m3u8_file) { int ret = ERROR_SUCCESS; @@ -799,7 +902,7 @@ int SrsHLS::_refresh_m3u8(int& fd, std::string m3u8_file) return ret; } -int SrsHLS::create_dir() +int SrsHls::create_dir() { int ret = ERROR_SUCCESS; @@ -822,41 +925,7 @@ int SrsHLS::create_dir() return ret; } -SrsTSMuxer::SrsTSMuxer() -{ - fd = -1; -} - -SrsTSMuxer::~SrsTSMuxer() -{ - close(); -} - -int SrsTSMuxer::open(std::string _path) -{ - int ret = ERROR_SUCCESS; - - path = _path; - - close(); - - int flags = O_CREAT|O_WRONLY|O_TRUNC; - mode_t mode = S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH; - if ((fd = ::open(path.c_str(), flags, mode)) < 0) { - ret = ERROR_HLS_OPEN_FAILED; - srs_error("open ts file %s failed. ret=%d", path.c_str(), ret); - return ret; - } - - // write mpegts header - if ((ret = SrsMpegtsWriter::write_header(fd)) != ERROR_SUCCESS) { - return ret; - } - - return ret; -} - -int SrsTSMuxer::write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer, SrsCodec* codec, SrsCodecSample* sample) +int SrsHls::write_audio() { int ret = ERROR_SUCCESS; @@ -920,14 +989,10 @@ int SrsTSMuxer::write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_b audio_buffer->append(buf->bytes, buf->size); } - if ((ret = SrsMpegtsWriter::write_frame(fd, audio_frame, audio_buffer)) != ERROR_SUCCESS) { - return ret; - } - return ret; } -int SrsTSMuxer::write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer, SrsCodec* codec, SrsCodecSample* sample) +int SrsHls::write_video() { int ret = ERROR_SUCCESS; @@ -991,6 +1056,81 @@ int SrsTSMuxer::write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_b video_buffer->append(buf->bytes, buf->size); } + return ret; +} + +int SrsHls::flush_audio() +{ + int ret = ERROR_SUCCESS; + + if (audio_buffer->size <= 0) { + return ret; + } + + if ((ret = current->muxer->write_audio(audio_frame, audio_buffer)) != ERROR_SUCCESS) { + return ret; + } + + // write success, clear and free the buffer + audio_buffer->free(); + + return ret; +} + +SrsTSMuxer::SrsTSMuxer() +{ + fd = -1; + _fresh = false; +} + +SrsTSMuxer::~SrsTSMuxer() +{ + close(); +} + +int SrsTSMuxer::open(std::string _path) +{ + int ret = ERROR_SUCCESS; + + path = _path; + + close(); + + int flags = O_CREAT|O_WRONLY|O_TRUNC; + mode_t mode = S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH; + if ((fd = ::open(path.c_str(), flags, mode)) < 0) { + ret = ERROR_HLS_OPEN_FAILED; + srs_error("open ts file %s failed. ret=%d", path.c_str(), ret); + return ret; + } + + // write mpegts header + if ((ret = SrsMpegtsWriter::write_header(fd)) != ERROR_SUCCESS) { + return ret; + } + + _fresh = true; + + return ret; +} + +int SrsTSMuxer::write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer) +{ + int ret = ERROR_SUCCESS; + + if ((ret = SrsMpegtsWriter::write_frame(fd, audio_frame, audio_buffer)) != ERROR_SUCCESS) { + return ret; + } + + _fresh = false; + + return ret; +} + +int SrsTSMuxer::write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer) +{ + int ret = ERROR_SUCCESS; + if ((ret = SrsMpegtsWriter::write_frame(fd, video_frame, video_buffer)) != ERROR_SUCCESS) { return ret; } @@ -1003,6 +1143,12 @@ void SrsTSMuxer::close() if (fd > 0) { ::close(fd); fd = -1; + _fresh = false; } } +bool SrsTSMuxer::fresh() +{ + return _fresh; +} + diff --git a/trunk/src/core/srs_core_hls.hpp b/trunk/src/core/srs_core_hls.hpp index e75b7198e..3e7a339be 100644 --- a/trunk/src/core/srs_core_hls.hpp +++ b/trunk/src/core/srs_core_hls.hpp @@ -64,10 +64,40 @@ struct SrsM3u8Segment virtual ~SrsM3u8Segment(); }; +/** +* jitter correct for audio, +* the sample rate 44100/32000 will lost precise, +* when mp4/ts(tbn=90000) covert to flv/rtmp(1000), +* so the Hls on ipad or iphone will corrupt, +* @see nginx-rtmp: est_pts +*/ +class SrsHlsAacJitter +{ +private: + int64_t base_pts; + int64_t nb_samples; + int sync_ms; +public: + SrsHlsAacJitter(); + virtual ~SrsHlsAacJitter(); + /** + * when buffer start, calc the "correct" pts for ts, + * @param flv_pts, the flv pts calc from flv header timestamp, + * @return the calc correct pts. + */ + virtual int64_t on_buffer_start(int64_t flv_pts, int sample_rate); + /** + * when buffer continue, muxer donot write to file, + * the audio buffer continue grow and donot need a pts, + * for the ts audio PES packet only has one pts at the first time. + */ + virtual void on_buffer_continue(); +}; + /** * write m3u8 hls. */ -class SrsHLS +class SrsHls { private: std::string vhost; @@ -95,16 +125,20 @@ private: SrsCodecBuffer* video_buffer; // last known dts int64_t stream_dts; + int64_t audio_buffer_start_pts; // last segment dts in m3u8 int64_t m3u8_dts; + // in ms, audio delay to flush the audios. + int64_t audio_delay; private: bool hls_enabled; SrsCodec* codec; SrsCodecSample* sample; SrsRtmpJitter* jitter; + SrsHlsAacJitter* aac_jitter; public: - SrsHLS(); - virtual ~SrsHLS(); + SrsHls(); + virtual ~SrsHls(); public: virtual int on_publish(std::string _vhost, std::string _app, std::string _stream); virtual void on_unpublish(); @@ -116,6 +150,10 @@ private: virtual int refresh_m3u8(); virtual int _refresh_m3u8(int& fd, std::string m3u8_file); virtual int create_dir(); +private: + virtual int write_audio(); + virtual int write_video(); + virtual int flush_audio(); }; class SrsTSMuxer @@ -123,14 +161,16 @@ class SrsTSMuxer private: int fd; std::string path; + bool _fresh; public: SrsTSMuxer(); virtual ~SrsTSMuxer(); public: virtual int open(std::string _path); - virtual int write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer, SrsCodec* codec, SrsCodecSample* sample); - virtual int write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer, SrsCodec* codec, SrsCodecSample* sample); + virtual int write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer); + virtual int write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer); virtual void close(); + virtual bool fresh(); }; #endif \ No newline at end of file diff --git a/trunk/src/core/srs_core_source.cpp b/trunk/src/core/srs_core_source.cpp index 560969eb2..92c9e8536 100644 --- a/trunk/src/core/srs_core_source.cpp +++ b/trunk/src/core/srs_core_source.cpp @@ -257,7 +257,7 @@ SrsSource* SrsSource::find(std::string stream_url) SrsSource::SrsSource(std::string _stream_url) { stream_url = _stream_url; - hls = new SrsHLS(); + hls = new SrsHls(); cache_metadata = cache_sh_video = cache_sh_audio = NULL; diff --git a/trunk/src/core/srs_core_source.hpp b/trunk/src/core/srs_core_source.hpp index b988b140a..e51382fa9 100644 --- a/trunk/src/core/srs_core_source.hpp +++ b/trunk/src/core/srs_core_source.hpp @@ -38,7 +38,7 @@ class SrsSource; class SrsCommonMessage; class SrsOnMetaDataPacket; class SrsSharedPtrMessage; -class SrsHLS; +class SrsHls; /** * time jitter detect and correct, @@ -125,7 +125,7 @@ public: */ static SrsSource* find(std::string stream_url); private: - SrsHLS* hls; + SrsHls* hls; std::string stream_url; std::vector consumers; // gop cache for client fast startup.