From b5bba29768f187880fc7c5ba90ea8c906194a6fd Mon Sep 17 00:00:00 2001 From: winlin Date: Mon, 2 Dec 2013 22:09:10 +0800 Subject: [PATCH] refine hls, extract ts cache for ts cache audio and flush once --- trunk/src/core/srs_core_hls.cpp | 434 ++++++++++++++++++-------------- trunk/src/core/srs_core_hls.hpp | 49 +++- 2 files changed, 281 insertions(+), 202 deletions(-) diff --git a/trunk/src/core/srs_core_hls.cpp b/trunk/src/core/srs_core_hls.cpp index 65f678150..1cca34985 100644 --- a/trunk/src/core/srs_core_hls.cpp +++ b/trunk/src/core/srs_core_hls.cpp @@ -856,13 +856,8 @@ int SrsM3u8Muxer::create_dir() return ret; } -SrsHls::SrsHls() +SrsTSCache::SrsTSCache() { - hls_enabled = false; - - codec = new SrsCodec(); - sample = new SrsCodecSample(); - jitter = new SrsRtmpJitter(); aac_jitter = new SrsHlsAacJitter(); ab = new SrsCodecBuffer(); @@ -870,15 +865,10 @@ SrsHls::SrsHls() af = new SrsMpegtsFrame(); vf = new SrsMpegtsFrame(); - - muxer = new SrsM3u8Muxer(); } -SrsHls::~SrsHls() +SrsTSCache::~SrsTSCache() { - srs_freep(codec); - srs_freep(sample); - srs_freep(jitter); srs_freep(aac_jitter); ab->free(); @@ -889,8 +879,237 @@ SrsHls::~SrsHls() srs_freep(af); srs_freep(vf); +} + +int SrsTSCache::write_audio(SrsCodec* codec, SrsM3u8Muxer* muxer, int64_t pts, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + // start buffer, set the af + if (ab->size == 0) { + pts = aac_jitter->on_buffer_start(pts, sample->sound_rate); + + af->dts = af->pts = audio_buffer_start_pts = pts; + af->pid = TS_AUDIO_PID; + af->sid = TS_AUDIO_AAC; + } else { + aac_jitter->on_buffer_continue(); + } + + // write audio to cache. + if ((ret = cache_audio(codec, sample)) != ERROR_SUCCESS) { + return ret; + } + + // flush if buffer exceed max size. + if (ab->size > SRS_HLS_AUDIO_CACHE_SIZE) { + if ((ret = muxer->flush_audio(af, ab)) != ERROR_SUCCESS) { + return ret; + } + } + // TODO: config it. + // in ms, audio delay to flush the audios. + int64_t audio_delay = SRS_CONF_DEFAULT_AAC_DELAY; + // flush if audio delay exceed + if (pts - audio_buffer_start_pts > audio_delay * 90) { + if ((ret = muxer->flush_audio(af, ab)) != ERROR_SUCCESS) { + return ret; + } + } + + return ret; +} + +int SrsTSCache::write_video(SrsCodec* codec, SrsM3u8Muxer* muxer, int64_t dts, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + // write video to cache. + if ((ret = cache_video(codec, sample)) != ERROR_SUCCESS) { + return ret; + } + + vf->dts = dts; + vf->pts = vf->dts + sample->cts * 90; + vf->pid = TS_VIDEO_PID; + vf->sid = TS_VIDEO_AVC; + vf->key = sample->frame_type == SrsCodecVideoAVCFrameKeyFrame; + + // flush video when got one + if ((ret = muxer->flush_video(af, ab, vf, vb)) != ERROR_SUCCESS) { + srs_error("m3u8 muxer flush video failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsTSCache::flush_audio(SrsM3u8Muxer* muxer) +{ + int ret = ERROR_SUCCESS; + + if ((ret = muxer->flush_audio(af, ab)) != ERROR_SUCCESS) { + srs_error("m3u8 muxer flush audio failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsTSCache::cache_audio(SrsCodec* codec, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + for (int i = 0; i < sample->nb_buffers; i++) { + SrsCodecBuffer* buf = &sample->buffers[i]; + int32_t size = buf->size; + + if (!buf->bytes || size <= 0 || size > 0x1fff) { + ret = ERROR_HLS_AAC_FRAME_LENGTH; + srs_error("invalid aac frame length=%d, ret=%d", size, ret); + return ret; + } + + // the frame length is the AAC raw data plus the adts header size. + int32_t frame_length = size + 7; + + // AAC-ADTS + // 6.2 Audio Data Transport Stream, ADTS + // in aac-iso-13818-7.pdf, page 26. + // fixed 7bytes header + static u_int8_t adts_header[7] = {0xff, 0xf1, 0x00, 0x00, 0x00, 0x0f, 0xfc}; + /* + // adts_fixed_header + // 2B, 16bits + int16_t syncword; //12bits, '1111 1111 1111' + int8_t ID; //1bit, '0' + int8_t layer; //2bits, '00' + int8_t protection_absent; //1bit, can be '1' + // 12bits + int8_t profile; //2bit, 7.1 Profiles, page 40 + TSAacSampleFrequency sampling_frequency_index; //4bits, Table 35, page 46 + int8_t private_bit; //1bit, can be '0' + int8_t channel_configuration; //3bits, Table 8 + int8_t original_or_copy; //1bit, can be '0' + int8_t home; //1bit, can be '0' + + // adts_variable_header + // 28bits + int8_t copyright_identification_bit; //1bit, can be '0' + int8_t copyright_identification_start; //1bit, can be '0' + int16_t frame_length; //13bits + int16_t adts_buffer_fullness; //11bits, 7FF signals that the bitstream is a variable rate bitstream. + int8_t number_of_raw_data_blocks_in_frame; //2bits, 0 indicating 1 raw_data_block() + */ + // profile, 2bits + adts_header[2] = (codec->aac_profile << 6) & 0xc0; + // sampling_frequency_index 4bits + adts_header[2] |= (codec->aac_sample_rate << 2) & 0x3c; + // channel_configuration 3bits + adts_header[2] |= (codec->aac_channels >> 2) & 0x01; + adts_header[3] = (codec->aac_channels << 6) & 0xc0; + // frame_length 13bits + adts_header[3] |= (frame_length >> 11) & 0x03; + adts_header[4] = (frame_length >> 3) & 0xff; + adts_header[5] = ((frame_length << 5) & 0xe0); + // adts_buffer_fullness; //11bits + adts_header[5] |= 0x1f; + + // copy to audio buffer + ab->append(adts_header, sizeof(adts_header)); + ab->append(buf->bytes, buf->size); + } + + return ret; +} + +int SrsTSCache::cache_video(SrsCodec* codec, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + static u_int8_t aud_nal[] = { 0x00, 0x00, 0x00, 0x01, 0x09, 0xf0 }; + vb->append(aud_nal, sizeof(aud_nal)); + + bool sps_pps_sent = false; + for (int i = 0; i < sample->nb_buffers; i++) { + SrsCodecBuffer* buf = &sample->buffers[i]; + int32_t size = buf->size; + + if (!buf->bytes || size <= 0) { + ret = ERROR_HLS_AVC_SAMPLE_SIZE; + srs_error("invalid avc sample length=%d, ret=%d", size, ret); + return ret; + } + + // 5bits, 7.3.1 NAL unit syntax, + // H.264-AVC-ISO_IEC_14496-10.pdf, page 44. + u_int8_t nal_unit_type; + nal_unit_type = *buf->bytes; + nal_unit_type &= 0x1f; + + // Table 7-1 – NAL unit type codes, page 61 + // 1: Coded slice + if (nal_unit_type == 1) { + sps_pps_sent = false; + } + // 5: Coded slice of an IDR picture. + // insert sps/pps before IDR or key frame is ok. + if (nal_unit_type == 5 && !sps_pps_sent) { + //if (vf->key && !sps_pps_sent) { + sps_pps_sent = true; + + // ngx_rtmp_hls_append_sps_pps + if (codec->sequenceParameterSetLength > 0) { + // AnnexB prefix + vb->append(aud_nal, 4); + // sps + vb->append(codec->sequenceParameterSetNALUnit, codec->sequenceParameterSetLength); + } + if (codec->pictureParameterSetLength > 0) { + // AnnexB prefix + vb->append(aud_nal, 4); + // pps + vb->append(codec->pictureParameterSetNALUnit, codec->pictureParameterSetLength); + } + } + + // sample start prefix, '00 00 00 01' or '00 00 01' + u_int8_t* p = aud_nal + 1; + u_int8_t* end = p + 3; + + // first AnnexB prefix is long (4 bytes) + if (i == 0) { + p = aud_nal; + } + vb->append(p, end - p); + + // sample data + vb->append(buf->bytes, buf->size); + } + + return ret; +} + +SrsHls::SrsHls() +{ + hls_enabled = false; + + codec = new SrsCodec(); + sample = new SrsCodecSample(); + jitter = new SrsRtmpJitter(); + + muxer = new SrsM3u8Muxer(); + ts_cache = new SrsTSCache(); +} + +SrsHls::~SrsHls() +{ + srs_freep(codec); + srs_freep(sample); + srs_freep(jitter); srs_freep(muxer); + srs_freep(ts_cache); } int SrsHls::on_publish(SrsRequest* req) @@ -953,7 +1172,7 @@ void SrsHls::on_unpublish() int ret = ERROR_SUCCESS; // close muxer when unpublish. - ret = muxer->flush_audio(af, ab); + ret = ts_cache->flush_audio(muxer); ret += muxer->segment_close(); if (ret != ERROR_SUCCESS) { srs_error("ignore m3u8 muxer flush/close audio failed. ret=%d", ret); @@ -1028,6 +1247,7 @@ int SrsHls::on_audio(SrsSharedPtrMessage* audio) sample->clear(); if ((ret = codec->audio_aac_demux(audio->payload, audio->size, sample)) != ERROR_SUCCESS) { + srs_error("codec demux audio failed. ret=%d", ret); return ret; } @@ -1042,44 +1262,18 @@ int SrsHls::on_audio(SrsSharedPtrMessage* audio) int64_t corrected_time = 0; if ((ret = jitter->correct(audio, 0, 0, &corrected_time)) != ERROR_SUCCESS) { + srs_error("rtmp jitter correct audio failed. ret=%d", ret); return ret; } // the pts calc from rtmp/flv header. int64_t pts = corrected_time * 90; - // start buffer, set the af - if (ab->size == 0) { - pts = aac_jitter->on_buffer_start(pts, sample->sound_rate); - - af->dts = af->pts = audio_buffer_start_pts = pts; - af->pid = TS_AUDIO_PID; - af->sid = TS_AUDIO_AAC; - } else { - aac_jitter->on_buffer_continue(); - } - - // write audio to cache. - if ((ret = cache_audio()) != ERROR_SUCCESS) { + if ((ret = ts_cache->write_audio(codec, muxer, pts, sample)) != ERROR_SUCCESS) { + srs_error("ts cache write audio failed. ret=%d", ret); return ret; } - // flush if buffer exceed max size. - if (ab->size > SRS_HLS_AUDIO_CACHE_SIZE) { - if ((ret = muxer->flush_audio(af, ab)) != ERROR_SUCCESS) { - return ret; - } - } - // TODO: config it. - // in ms, audio delay to flush the audios. - int64_t audio_delay = SRS_CONF_DEFAULT_AAC_DELAY; - // flush if audio delay exceed - if (pts - audio_buffer_start_pts > audio_delay * 90) { - if ((ret = muxer->flush_audio(af, ab)) != ERROR_SUCCESS) { - return ret; - } - } - return ret; } @@ -1096,6 +1290,7 @@ int SrsHls::on_video(SrsSharedPtrMessage* video) sample->clear(); if ((ret = codec->video_avc_demux(video->payload, video->size, sample)) != ERROR_SUCCESS) { + srs_error("codec demux video failed. ret=%d", ret); return ret; } @@ -1104,166 +1299,23 @@ int SrsHls::on_video(SrsSharedPtrMessage* video) } // ignore sequence header - if (sample->frame_type == SrsCodecVideoAVCFrameKeyFrame && sample->avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) { + if (sample->frame_type == SrsCodecVideoAVCFrameKeyFrame + && sample->avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) { return ret; } int64_t corrected_time = 0; if ((ret = jitter->correct(video, 0, 0, &corrected_time)) != ERROR_SUCCESS) { + srs_error("rtmp jitter correct video failed. ret=%d", ret); return ret; } - // write video to cache. - if ((ret = cache_video()) != ERROR_SUCCESS) { + int64_t dts = corrected_time * 90; + if ((ret = ts_cache->write_video(codec, muxer, dts, sample)) != ERROR_SUCCESS) { + srs_error("ts cache write video failed. ret=%d", ret); return ret; } - vf->dts = corrected_time * 90; - vf->pts = vf->dts + sample->cts * 90; - vf->pid = TS_VIDEO_PID; - vf->sid = TS_VIDEO_AVC; - vf->key = sample->frame_type == SrsCodecVideoAVCFrameKeyFrame; - - // flush video when got one - if ((ret = muxer->flush_video(af, ab, vf, vb)) != ERROR_SUCCESS) { - srs_error("m3u8 muxer flush video failed. ret=%d", ret); - return ret; - } - - return ret; -} - -int SrsHls::cache_audio() -{ - int ret = ERROR_SUCCESS; - - for (int i = 0; i < sample->nb_buffers; i++) { - SrsCodecBuffer* buf = &sample->buffers[i]; - int32_t size = buf->size; - - if (!buf->bytes || size <= 0 || size > 0x1fff) { - ret = ERROR_HLS_AAC_FRAME_LENGTH; - srs_error("invalid aac frame length=%d, ret=%d", size, ret); - return ret; - } - - // the frame length is the AAC raw data plus the adts header size. - int32_t frame_length = size + 7; - - // AAC-ADTS - // 6.2 Audio Data Transport Stream, ADTS - // in aac-iso-13818-7.pdf, page 26. - // fixed 7bytes header - static u_int8_t adts_header[7] = {0xff, 0xf1, 0x00, 0x00, 0x00, 0x0f, 0xfc}; - /* - // adts_fixed_header - // 2B, 16bits - int16_t syncword; //12bits, '1111 1111 1111' - int8_t ID; //1bit, '0' - int8_t layer; //2bits, '00' - int8_t protection_absent; //1bit, can be '1' - // 12bits - int8_t profile; //2bit, 7.1 Profiles, page 40 - TSAacSampleFrequency sampling_frequency_index; //4bits, Table 35, page 46 - int8_t private_bit; //1bit, can be '0' - int8_t channel_configuration; //3bits, Table 8 - int8_t original_or_copy; //1bit, can be '0' - int8_t home; //1bit, can be '0' - - // adts_variable_header - // 28bits - int8_t copyright_identification_bit; //1bit, can be '0' - int8_t copyright_identification_start; //1bit, can be '0' - int16_t frame_length; //13bits - int16_t adts_buffer_fullness; //11bits, 7FF signals that the bitstream is a variable rate bitstream. - int8_t number_of_raw_data_blocks_in_frame; //2bits, 0 indicating 1 raw_data_block() - */ - // profile, 2bits - adts_header[2] = (codec->aac_profile << 6) & 0xc0; - // sampling_frequency_index 4bits - adts_header[2] |= (codec->aac_sample_rate << 2) & 0x3c; - // channel_configuration 3bits - adts_header[2] |= (codec->aac_channels >> 2) & 0x01; - adts_header[3] = (codec->aac_channels << 6) & 0xc0; - // frame_length 13bits - adts_header[3] |= (frame_length >> 11) & 0x03; - adts_header[4] = (frame_length >> 3) & 0xff; - adts_header[5] = ((frame_length << 5) & 0xe0); - // adts_buffer_fullness; //11bits - adts_header[5] |= 0x1f; - - // copy to audio buffer - ab->append(adts_header, sizeof(adts_header)); - ab->append(buf->bytes, buf->size); - } - - return ret; -} - -int SrsHls::cache_video() -{ - int ret = ERROR_SUCCESS; - - static u_int8_t aud_nal[] = { 0x00, 0x00, 0x00, 0x01, 0x09, 0xf0 }; - vb->append(aud_nal, sizeof(aud_nal)); - - bool sps_pps_sent = false; - for (int i = 0; i < sample->nb_buffers; i++) { - SrsCodecBuffer* buf = &sample->buffers[i]; - int32_t size = buf->size; - - if (!buf->bytes || size <= 0) { - ret = ERROR_HLS_AVC_SAMPLE_SIZE; - srs_error("invalid avc sample length=%d, ret=%d", size, ret); - return ret; - } - - // 5bits, 7.3.1 NAL unit syntax, - // H.264-AVC-ISO_IEC_14496-10.pdf, page 44. - u_int8_t nal_unit_type; - nal_unit_type = *buf->bytes; - nal_unit_type &= 0x1f; - - // Table 7-1 – NAL unit type codes, page 61 - // 1: Coded slice - if (nal_unit_type == 1) { - sps_pps_sent = false; - } - // 5: Coded slice of an IDR picture. - // insert sps/pps before IDR or key frame is ok. - if (nal_unit_type == 5 && !sps_pps_sent) { - //if (vf->key && !sps_pps_sent) { - sps_pps_sent = true; - - // ngx_rtmp_hls_append_sps_pps - if (codec->sequenceParameterSetLength > 0) { - // AnnexB prefix - vb->append(aud_nal, 4); - // sps - vb->append(codec->sequenceParameterSetNALUnit, codec->sequenceParameterSetLength); - } - if (codec->pictureParameterSetLength > 0) { - // AnnexB prefix - vb->append(aud_nal, 4); - // pps - vb->append(codec->pictureParameterSetNALUnit, codec->pictureParameterSetLength); - } - } - - // sample start prefix, '00 00 00 01' or '00 00 01' - u_int8_t* p = aud_nal + 1; - u_int8_t* end = p + 3; - - // first AnnexB prefix is long (4 bytes) - if (i == 0) { - p = aud_nal; - } - vb->append(p, end - p); - - // sample data - vb->append(buf->bytes, buf->size); - } - return ret; } diff --git a/trunk/src/core/srs_core_hls.hpp b/trunk/src/core/srs_core_hls.hpp index 06749cfac..0a8aa79d6 100644 --- a/trunk/src/core/srs_core_hls.hpp +++ b/trunk/src/core/srs_core_hls.hpp @@ -159,6 +159,43 @@ private: virtual int create_dir(); }; +/** +* ts need to cache some audio then flush +*/ +class SrsTSCache +{ +private: + // current frame and buffer + SrsMpegtsFrame* af; + SrsCodecBuffer* ab; + SrsMpegtsFrame* vf; + SrsCodecBuffer* vb; +private: + // the audio cache buffer start pts, to flush audio if full. + int64_t audio_buffer_start_pts; + // time jitter for aac + SrsHlsAacJitter* aac_jitter; +public: + SrsTSCache(); + virtual ~SrsTSCache(); +public: + /** + * write audio to cache, if need to flush, flush to muxer. + */ + virtual int write_audio(SrsCodec* codec, SrsM3u8Muxer* muxer, int64_t pts, SrsCodecSample* sample); + /** + * write video to muxer. + */ + virtual int write_video(SrsCodec* codec, SrsM3u8Muxer* muxer, int64_t dts, SrsCodecSample* sample); + /** + * flush audio in cache to muxer. + */ + virtual int flush_audio(SrsM3u8Muxer* muxer); +private: + virtual int cache_audio(SrsCodec* codec, SrsCodecSample* sample); + virtual int cache_video(SrsCodec* codec, SrsCodecSample* sample); +}; + /** * write m3u8 hls. */ @@ -166,19 +203,12 @@ class SrsHls { private: SrsM3u8Muxer* muxer; - // current frame and buffer - SrsMpegtsFrame* af; - SrsCodecBuffer* ab; - SrsMpegtsFrame* vf; - SrsCodecBuffer* vb; - // the audio cache buffer start pts, to flush audio if full. - int64_t audio_buffer_start_pts; + SrsTSCache* ts_cache; private: bool hls_enabled; SrsCodec* codec; SrsCodecSample* sample; SrsRtmpJitter* jitter; - SrsHlsAacJitter* aac_jitter; public: SrsHls(); virtual ~SrsHls(); @@ -188,9 +218,6 @@ public: virtual int on_meta_data(SrsOnMetaDataPacket* metadata); virtual int on_audio(SrsSharedPtrMessage* audio); virtual int on_video(SrsSharedPtrMessage* video); -private: - virtual int cache_audio(); - virtual int cache_video(); }; #endif