From 2c42350489cb2b744f5a8878252359e69e80b0f4 Mon Sep 17 00:00:00 2001 From: winlin Date: Sun, 25 Jan 2015 16:42:22 +0800 Subject: [PATCH] for #301, http ts stream support h.264+mp3. 2.0.106 --- README.md | 1 + trunk/src/app/srs_app_hls.cpp | 31 ++++- trunk/src/app/srs_app_hls.hpp | 6 +- trunk/src/app/srs_app_source.cpp | 27 ++-- trunk/src/core/srs_core.hpp | 2 +- trunk/src/kernel/srs_kernel_avc.cpp | 89 ++++++++---- trunk/src/kernel/srs_kernel_avc.hpp | 3 + trunk/src/kernel/srs_kernel_error.hpp | 1 + trunk/src/kernel/srs_kernel_ts.cpp | 186 +++++++++++++++++++++----- trunk/src/kernel/srs_kernel_ts.hpp | 34 +++++ 10 files changed, 305 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index e7ddd47af..3c9291dac 100755 --- a/README.md +++ b/README.md @@ -521,6 +521,7 @@ Supported operating systems and hardware: ### SRS 2.0 history +* v2.0, 2015-01-25, for [#301](https://github.com/winlinvip/simple-rtmp-server/issues/301), http ts stream support h.264+mp3. 2.0.106 * v2.0, 2015-01-25, hotfix [#268](https://github.com/winlinvip/simple-rtmp-server/issues/268), refine the pcr start at 0, dts/pts plus delay. 2.0.105 * v2.0, 2015-01-25, hotfix [#151](https://github.com/winlinvip/simple-rtmp-server/issues/151), refine pcr=dts-800ms and use dts/pts directly. 2.0.104 * v2.0, 2015-01-23, hotfix [#151](https://github.com/winlinvip/simple-rtmp-server/issues/151), use absolutely overflow to make jwplayer happy. 2.0.103 diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 8b1dbda82..30ce65943 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -199,6 +199,13 @@ bool SrsHlsMuxer::is_segment_absolutely_overflow() return current->duration >= 2 * hls_fragment; } +int SrsHlsMuxer::update_acodec(SrsCodecAudio acodec) +{ + srs_assert(current); + srs_assert(current->muxer); + return current->muxer->update_acodec(acodec); +} + int SrsHlsMuxer::flush_audio(SrsMpegtsFrame* af, SrsSimpleBuffer* ab) { int ret = ERROR_SUCCESS; @@ -572,8 +579,6 @@ int SrsHlsCache::on_sequence_header(SrsHlsMuxer* muxer) int SrsHlsCache::write_audio(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t pts, SrsCodecSample* sample) { int ret = ERROR_SUCCESS; - - audio_buffer_start_pts = pts; // write audio to cache. if ((ret = cache->cache_audio(codec, pts, sample)) != ERROR_SUCCESS) { @@ -591,7 +596,7 @@ int SrsHlsCache::write_audio(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t // in ms, audio delay to flush the audios. int64_t audio_delay = SRS_CONF_DEFAULT_AAC_DELAY; // flush if audio delay exceed - if (pts - audio_buffer_start_pts > audio_delay * 90) { + if (pts - cache->audio_buffer_start_pts > audio_delay * 90) { if ((ret = muxer->flush_audio(cache->af, cache->ab)) != ERROR_SUCCESS) { return ret; } @@ -773,11 +778,25 @@ int SrsHls::on_audio(SrsSharedPtrMessage* __audio) sample->clear(); if ((ret = codec->audio_aac_demux(audio->payload, audio->size, sample)) != ERROR_SUCCESS) { - srs_error("hls codec demux audio failed. ret=%d", ret); + if (ret != ERROR_HLS_TRY_MP3) { + srs_error("hls aac demux audio failed. ret=%d", ret); + return ret; + } + if ((ret = codec->audio_mp3_demux(audio->payload, audio->size, sample)) != ERROR_SUCCESS) { + srs_error("hls mp3 demux audio failed. ret=%d", ret); + return ret; + } + } + SrsCodecAudio acodec = (SrsCodecAudio)codec->audio_codec_id; + + // ts support audio codec: aac/mp3 + if (acodec != SrsCodecAudioAAC && acodec != SrsCodecAudioMP3) { return ret; } - - if (codec->audio_codec_id != SrsCodecAudioAAC) { + + // when codec changed, write new header. + if ((ret = muxer->update_acodec(acodec)) != ERROR_SUCCESS) { + srs_error("http: ts audio write header failed. ret=%d", ret); return ret; } diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp index 3fbef0f33..2fe45ec93 100644 --- a/trunk/src/app/srs_app_hls.hpp +++ b/trunk/src/app/srs_app_hls.hpp @@ -37,6 +37,8 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include +#include + class SrsSharedPtrMessage; class SrsCodecSample; class SrsMpegtsFrame; @@ -141,6 +143,8 @@ public: * @see https://github.com/winlinvip/simple-rtmp-server/issues/151#issuecomment-71155184 */ virtual bool is_segment_absolutely_overflow(); +public: + virtual int update_acodec(SrsCodecAudio acodec); virtual int flush_audio(SrsMpegtsFrame* af, SrsSimpleBuffer* ab); virtual int flush_video(SrsMpegtsFrame* af, SrsSimpleBuffer* ab, SrsMpegtsFrame* vf, SrsSimpleBuffer* vb); /** @@ -174,8 +178,6 @@ private: class SrsHlsCache { private: - // the audio cache buffer start pts, to flush audio if full. - int64_t audio_buffer_start_pts; SrsTsCache* cache; public: SrsHlsCache(); diff --git a/trunk/src/app/srs_app_source.cpp b/trunk/src/app/srs_app_source.cpp index 59e203830..10a8bfb33 100644 --- a/trunk/src/app/srs_app_source.cpp +++ b/trunk/src/app/srs_app_source.cpp @@ -1349,12 +1349,17 @@ int SrsSource::on_audio(SrsCommonMessage* __audio) } } - // cache the sequence header if h264 - // donot cache the sequence header to gop_cache, return here. - if (SrsFlvCodec::audio_is_sequence_header(msg.payload, msg.size)) { + // cache the sequence header of aac, or first packet of mp3. + // for example, the mp3 is used for hls to write the "right" audio codec. + bool is_aac_sequence_header = SrsFlvCodec::audio_is_sequence_header(msg.payload, msg.size); + if (is_aac_sequence_header || !cache_sh_audio) { srs_freep(cache_sh_audio); cache_sh_audio = msg.copy(); - + } + + // cache the sequence header if aac + // donot cache the sequence header to gop_cache, return here. + if (is_aac_sequence_header) { // parse detail audio codec SrsAvcAacCodec codec; SrsCodecSample sample; @@ -1768,18 +1773,20 @@ int SrsSource::create_consumer(SrsConsumer*& consumer, bool ds, bool dm, bool dg srs_info("dispatch metadata success"); // copy sequence header + // copy audio sequence first, for hls to fast parse the "right" audio codec. + // @see https://github.com/winlinvip/simple-rtmp-server/issues/301 + if (ds && cache_sh_audio && (ret = consumer->enqueue(cache_sh_audio, atc, tba, tbv, ag)) != ERROR_SUCCESS) { + srs_error("dispatch audio sequence header failed. ret=%d", ret); + return ret; + } + srs_info("dispatch audio sequence header success"); + if (ds && cache_sh_video && (ret = consumer->enqueue(cache_sh_video, atc, tba, tbv, ag)) != ERROR_SUCCESS) { srs_error("dispatch video sequence header failed. ret=%d", ret); return ret; } srs_info("dispatch video sequence header success"); - if (cache_sh_audio && (ret = consumer->enqueue(cache_sh_audio, atc, tba, tbv, ag)) != ERROR_SUCCESS) { - srs_error("dispatch audio sequence header failed. ret=%d", ret); - return ret; - } - srs_info("dispatch audio sequence header success"); - // copy gop cache to client. if (dg && (ret = gop_cache->dump(consumer, atc, tba, tbv, ag)) != ERROR_SUCCESS) { return ret; diff --git a/trunk/src/core/srs_core.hpp b/trunk/src/core/srs_core.hpp index 3e78c335f..ac75ff4ed 100644 --- a/trunk/src/core/srs_core.hpp +++ b/trunk/src/core/srs_core.hpp @@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // current release version #define VERSION_MAJOR 2 #define VERSION_MINOR 0 -#define VERSION_REVISION 105 +#define VERSION_REVISION 106 // server info. #define RTMP_SIG_SRS_KEY "SRS" diff --git a/trunk/src/kernel/srs_kernel_avc.cpp b/trunk/src/kernel/srs_kernel_avc.cpp index 9e0b6c1ab..8dc9112a3 100644 --- a/trunk/src/kernel/srs_kernel_avc.cpp +++ b/trunk/src/kernel/srs_kernel_avc.cpp @@ -60,6 +60,7 @@ void SrsCodecSample::clear() frame_type = SrsCodecVideoAVCFrameReserved; avc_packet_type = SrsCodecVideoAVCTypeReserved; + acodec = SrsCodecAudioReserved1; sound_rate = SrsCodecAudioSampleRateReserved; sound_size = SrsCodecAudioSampleSizeReserved; sound_type = SrsCodecAudioSoundTypeReserved; @@ -91,10 +92,13 @@ SrsAvcAacCodec::SrsAvcAacCodec() duration = 0; NAL_unit_length = 0; frame_rate = 0; + video_data_rate = 0; video_codec_id = 0; + audio_data_rate = 0; audio_codec_id = 0; + avc_profile = 0; avc_level = 0; aac_profile = 0; @@ -104,6 +108,7 @@ SrsAvcAacCodec::SrsAvcAacCodec() avc_extra_data = NULL; aac_extra_size = 0; aac_extra_data = NULL; + sequenceParameterSetLength = 0; sequenceParameterSetNALUnit = NULL; pictureParameterSetLength = 0; @@ -129,7 +134,7 @@ int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample sample->is_video = false; if (!data || size <= 0) { - srs_trace("no audio present, hls ignore it."); + srs_trace("no audio present, ignore it."); return ret; } @@ -140,7 +145,7 @@ int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample // audio decode if (!stream->require(1)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode audio sound_format failed. ret=%d", ret); + srs_error("audio codec decode sound_format failed. ret=%d", ret); return ret; } @@ -153,20 +158,27 @@ int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample sound_format = (sound_format >> 4) & 0x0f; audio_codec_id = sound_format; + sample->acodec = (SrsCodecAudio)audio_codec_id; + sample->sound_type = (SrsCodecAudioSoundType)sound_type; sample->sound_rate = (SrsCodecAudioSampleRate)sound_rate; sample->sound_size = (SrsCodecAudioSampleSize)sound_size; + + // we support h.264+mp3 for hls. + if (audio_codec_id == SrsCodecAudioMP3) { + return ERROR_HLS_TRY_MP3; + } // only support aac if (audio_codec_id != SrsCodecAudioAAC) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls only support audio aac codec. actual=%d, ret=%d", audio_codec_id, ret); + srs_error("audio codec only support mp3/aac codec. actual=%d, ret=%d", audio_codec_id, ret); return ret; } if (!stream->require(1)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode audio aac_packet_type failed. ret=%d", ret); + srs_error("audio codec decode aac_packet_type failed. ret=%d", ret); return ret; } @@ -189,7 +201,7 @@ int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample // channelConfiguration, aac_channels, 4bits if (!stream->require(2)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode audio aac sequence header failed. ret=%d", ret); + srs_error("audio codec decode aac sequence header failed. ret=%d", ret); return ret; } aac_profile = stream->read_1bytes(); @@ -201,7 +213,7 @@ int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample if (aac_profile == 0 || aac_profile == 0x1f) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode audio aac sequence header failed, " + srs_error("audio codec decode aac sequence header failed, " "adts object=%d invalid. ret=%d", aac_profile, ret); return ret; } @@ -221,14 +233,14 @@ int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample // ensure the sequence header demuxed if (aac_extra_size <= 0 || !aac_extra_data) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode audio aac failed, sequence header not found. ret=%d", ret); + srs_error("audio codec decode aac failed, sequence header not found. ret=%d", ret); return ret; } // Raw AAC frame data in UI8 [] // 6.3 Raw Data, aac-iso-13818-7.pdf, page 28 if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), stream->size() - stream->pos())) != ERROR_SUCCESS) { - srs_error("hls add audio sample failed. ret=%d", ret); + srs_error("audio codec add sample failed. ret=%d", ret); return ret; } } else { @@ -264,6 +276,31 @@ int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample return ret; } +int SrsAvcAacCodec::audio_mp3_demux(char* data, int size, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + // we always decode aac then mp3. + srs_assert(sample->acodec == SrsCodecAudioMP3); + + // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76 + if (!data || size <= 1) { + srs_trace("no mp3 audio present, ignore it."); + return ret; + } + + // mp3 payload. + if ((ret = sample->add_sample_unit(data + 1, size - 1)) != ERROR_SUCCESS) { + srs_error("audio codec add mp3 sample failed. ret=%d", ret); + return ret; + } + + srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", + sample->sound_type, audio_codec_id, sample->sound_size, sample->sound_rate, sample->acodec, size); + + return ret; +} + int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample) { int ret = ERROR_SUCCESS; @@ -271,7 +308,7 @@ int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample sample->is_video = true; if (!data || size <= 0) { - srs_trace("no video present, hls ignore it."); + srs_trace("no video present, ignore it."); return ret; } @@ -282,7 +319,7 @@ int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample // video decode if (!stream->require(1)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video frame_type failed. ret=%d", ret); + srs_error("video codec decode frame_type failed. ret=%d", ret); return ret; } @@ -296,21 +333,21 @@ int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample // ignore info frame without error, // @see https://github.com/winlinvip/simple-rtmp-server/issues/288#issuecomment-69863909 if (sample->frame_type == SrsCodecVideoAVCFrameVideoInfoFrame) { - srs_warn("hls igone the info frame, ret=%d", ret); + srs_warn("video codec igone the info frame, ret=%d", ret); return ret; } // only support h.264/avc if (codec_id != SrsCodecVideoAVC) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls only support video h.264/avc codec. actual=%d, ret=%d", codec_id, ret); + srs_error("video codec only support video h.264/avc codec. actual=%d, ret=%d", codec_id, ret); return ret; } video_codec_id = codec_id; if (!stream->require(4)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc_packet_type failed. ret=%d", ret); + srs_error("video codec decode avc_packet_type failed. ret=%d", ret); return ret; } int8_t avc_packet_type = stream->read_1bytes(); @@ -328,7 +365,7 @@ int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample // ensure the sequence header demuxed if (avc_extra_size <= 0 || !avc_extra_data) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc failed, sequence header not found. ret=%d", ret); + srs_error("avc decode failed, sequence header not found. ret=%d", ret); return ret; } @@ -371,7 +408,7 @@ int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) if (!stream->require(6)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header failed. ret=%d", ret); + srs_error("avc decode sequenc header failed. ret=%d", ret); return ret; } //int8_t configurationVersion = stream->read_1bytes(); @@ -402,25 +439,25 @@ int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) // 1 sps if (!stream->require(1)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header sps failed. ret=%d", ret); + srs_error("avc decode sequenc header sps failed. ret=%d", ret); return ret; } int8_t numOfSequenceParameterSets = stream->read_1bytes(); numOfSequenceParameterSets &= 0x1f; if (numOfSequenceParameterSets != 1) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header sps failed. ret=%d", ret); + srs_error("avc decode sequenc header sps failed. ret=%d", ret); return ret; } if (!stream->require(2)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header sps size failed. ret=%d", ret); + srs_error("avc decode sequenc header sps size failed. ret=%d", ret); return ret; } sequenceParameterSetLength = stream->read_2bytes(); if (!stream->require(sequenceParameterSetLength)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header sps data failed. ret=%d", ret); + srs_error("avc decode sequenc header sps data failed. ret=%d", ret); return ret; } if (sequenceParameterSetLength > 0) { @@ -432,25 +469,25 @@ int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) // 1 pps if (!stream->require(1)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header pps failed. ret=%d", ret); + srs_error("avc decode sequenc header pps failed. ret=%d", ret); return ret; } int8_t numOfPictureParameterSets = stream->read_1bytes(); numOfPictureParameterSets &= 0x1f; if (numOfPictureParameterSets != 1) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header pps failed. ret=%d", ret); + srs_error("avc decode sequenc header pps failed. ret=%d", ret); return ret; } if (!stream->require(2)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header pps size failed. ret=%d", ret); + srs_error("avc decode sequenc header pps size failed. ret=%d", ret); return ret; } pictureParameterSetLength = stream->read_2bytes(); if (!stream->require(pictureParameterSetLength)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc sequenc header pps data failed. ret=%d", ret); + srs_error("avc decode sequenc header pps data failed. ret=%d", ret); return ret; } if (pictureParameterSetLength > 0) { @@ -534,7 +571,7 @@ int SrsAvcAacCodec::avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sam // unsigned int((NAL_unit_length+1)*8) NALUnitLength; if (!stream->require(NAL_unit_length + 1)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc NALU size failed. ret=%d", ret); + srs_error("avc decode NALU size failed. ret=%d", ret); return ret; } int32_t NALUnitLength = 0; @@ -557,12 +594,12 @@ int SrsAvcAacCodec::avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sam // NALUnit if (!stream->require(NALUnitLength)) { ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode video avc NALU data failed. ret=%d", ret); + srs_error("avc decode NALU data failed. ret=%d", ret); return ret; } // 7.3.1 NAL unit syntax, H.264-AVC-ISO_IEC_14496-10.pdf, page 44. if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), NALUnitLength)) != ERROR_SUCCESS) { - srs_error("hls add video sample failed. ret=%d", ret); + srs_error("avc add video sample failed. ret=%d", ret); return ret; } stream->skip(NALUnitLength); diff --git a/trunk/src/kernel/srs_kernel_avc.hpp b/trunk/src/kernel/srs_kernel_avc.hpp index d53f370fd..126e865f2 100644 --- a/trunk/src/kernel/srs_kernel_avc.hpp +++ b/trunk/src/kernel/srs_kernel_avc.hpp @@ -156,6 +156,8 @@ public: SrsCodecVideoAVCType avc_packet_type; public: // audio specified + SrsCodecAudio acodec; + // audio aac specified. SrsCodecAudioSampleRate sound_rate; SrsCodecAudioSampleSize sound_size; SrsCodecAudioSoundType sound_type; @@ -271,6 +273,7 @@ public: * demux the aac raw to sample units. */ virtual int audio_aac_demux(char* data, int size, SrsCodecSample* sample); + virtual int audio_mp3_demux(char* data, int size, SrsCodecSample* sample); /** * demux the video packet in h.264 codec. * the packet mux in FLV/RTMP format defined in flv specification. diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 9303ba429..7c2677ef9 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -202,6 +202,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_AAC_REQUIRED_ADTS 3046 #define ERROR_AAC_ADTS_HEADER 3047 #define ERROR_AAC_DATA_INVALID 3048 +#define ERROR_HLS_TRY_MP3 3049 /////////////////////////////////////////////////////// // HTTP/StreamCaster protocol error. diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index dbf5a22b2..6f3da8910 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -53,6 +53,7 @@ using namespace std; // ts aac stream id. #define TS_AUDIO_AAC 0xc0 +#define TS_AUDIO_MP3 0x04 // ts avc stream id. #define TS_VIDEO_AVC 0xe0 @@ -118,11 +119,18 @@ u_int8_t mpegts_header[] = { // must generate header with/without video, @see: // https://github.com/winlinvip/simple-rtmp-server/issues/40 0x1b, 0xe1, 0x00, 0xf0, 0x00, /* h264, pid=0x100=256 */ +}; +u_int8_t mpegts_header_aac[] = { 0x0f, 0xe1, 0x01, 0xf0, 0x00, /* aac, pid=0x101=257 */ - /*0x03, 0xe1, 0x01, 0xf0, 0x00,*/ /* mp3 */ /* CRC */ 0x2f, 0x44, 0xb9, 0x9b, /* crc for aac */ - /*0x4e, 0x59, 0x3d, 0x1e,*/ /* crc for mp3 */ +}; +u_int8_t mpegts_header_mp3[] = { + 0x03, 0xe1, 0x01, 0xf0, 0x00, /* mp3 */ + /* CRC */ + 0x4e, 0x59, 0x3d, 0x1e, /* crc for mp3 */ +}; +u_int8_t mpegts_header_padding[] = { /* stuffing 157 bytes */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -147,7 +155,7 @@ u_int8_t mpegts_header[] = { class SrsMpegtsWriter { public: - static int write_header(SrsFileWriter* writer) + static int write_header(SrsFileWriter* writer, SrsCodecAudio acodec) { int ret = ERROR_SUCCESS; @@ -157,6 +165,26 @@ public: return ret; } + if (acodec == SrsCodecAudioAAC) { + if ((ret = writer->write(mpegts_header_aac, sizeof(mpegts_header_aac), NULL)) != ERROR_SUCCESS) { + ret = ERROR_HLS_WRITE_FAILED; + srs_error("write ts file aac header failed. ret=%d", ret); + return ret; + } + } else { + if ((ret = writer->write(mpegts_header_mp3, sizeof(mpegts_header_mp3), NULL)) != ERROR_SUCCESS) { + ret = ERROR_HLS_WRITE_FAILED; + srs_error("write ts file mp3 header failed. ret=%d", ret); + return ret; + } + } + + if ((ret = writer->write(mpegts_header_padding, sizeof(mpegts_header_padding), NULL)) != ERROR_SUCCESS) { + ret = ERROR_HLS_WRITE_FAILED; + srs_error("write ts file padding header failed. ret=%d", ret); + return ret; + } + return ret; } static int write_frame(SrsFileWriter* writer, SrsMpegtsFrame* frame, SrsSimpleBuffer* buffer) @@ -375,6 +403,11 @@ SrsMpegtsFrame::SrsMpegtsFrame() SrsTSMuxer::SrsTSMuxer(SrsFileWriter* w) { writer = w; + + // reserved is not written. + previous = SrsCodecAudioReserved1; + // current default to aac. + current = SrsCodecAudioAAC; } SrsTSMuxer::~SrsTSMuxer() @@ -393,12 +426,19 @@ int SrsTSMuxer::open(string _path) if ((ret = writer->open(path)) != ERROR_SUCCESS) { return ret; } + + return ret; +} - // write mpegts header - if ((ret = SrsMpegtsWriter::write_header(writer)) != ERROR_SUCCESS) { +int SrsTSMuxer::update_acodec(SrsCodecAudio acodec) +{ + int ret = ERROR_SUCCESS; + + if (current == acodec) { return ret; } - + current = acodec; + return ret; } @@ -406,6 +446,14 @@ int SrsTSMuxer::write_audio(SrsMpegtsFrame* af, SrsSimpleBuffer* ab) { int ret = ERROR_SUCCESS; + // when acodec changed, write header. + if (current != previous) { + previous = current; + if ((ret = SrsMpegtsWriter::write_header(writer, previous)) != ERROR_SUCCESS) { + return ret; + } + } + if ((ret = SrsMpegtsWriter::write_frame(writer, af, ab)) != ERROR_SUCCESS) { return ret; } @@ -417,6 +465,14 @@ int SrsTSMuxer::write_video(SrsMpegtsFrame* vf, SrsSimpleBuffer* vb) { int ret = ERROR_SUCCESS; + // when acodec changed, write header. + if (current != previous) { + previous = current; + if ((ret = SrsMpegtsWriter::write_header(writer, previous)) != ERROR_SUCCESS) { + return ret; + } + } + if ((ret = SrsMpegtsWriter::write_frame(writer, vf, vb)) != ERROR_SUCCESS) { return ret; } @@ -501,6 +557,8 @@ SrsTsCache::SrsTsCache() af = new SrsMpegtsFrame(); vf = new SrsMpegtsFrame(); + + audio_buffer_start_pts = 0; } SrsTsCache::~SrsTsCache() @@ -520,23 +578,53 @@ SrsTsCache::~SrsTsCache() int SrsTsCache::cache_audio(SrsAvcAacCodec* codec, int64_t pts, SrsCodecSample* sample) { int ret = ERROR_SUCCESS; - - // start buffer, set the af + + // @remark, always use the orignal pts. if (ab->length() == 0) { - pts = aac_jitter->on_buffer_start(pts, sample->sound_rate, codec->aac_sample_rate); - - af->dts = af->pts = pts; - af->pid = TS_AUDIO_PID; - af->sid = TS_AUDIO_AAC; - } else { - aac_jitter->on_buffer_continue(); + audio_buffer_start_pts = pts; } - // write audio to cache. - if ((ret = do_cache_audio(codec, sample)) != ERROR_SUCCESS) { + // must be aac or mp3 + SrsCodecAudio acodec = (SrsCodecAudio)codec->audio_codec_id; + srs_assert(acodec == SrsCodecAudioAAC || acodec == SrsCodecAudioMP3); + + // cache the aac audio. + if (codec->audio_codec_id == SrsCodecAudioAAC) { + // for aac audio, recalc the timestamp by aac jitter. + if (ab->length() == 0) { + pts = aac_jitter->on_buffer_start(pts, sample->sound_rate, codec->aac_sample_rate); + + af->dts = af->pts = pts; + af->pid = TS_AUDIO_PID; + af->sid = TS_AUDIO_AAC; + } else { + aac_jitter->on_buffer_continue(); + } + + // write aac audio to cache. + if ((ret = do_cache_audio(codec, sample)) != ERROR_SUCCESS) { + return ret; + } + return ret; } + // cache the mp3 audio. + if (codec->audio_codec_id == SrsCodecAudioMP3) { + // for mp3 audio, recalc the timestamp by mp3 jitter. + // TODO: FIXME: implements it. + af->dts = af->pts = pts; + af->pid = TS_AUDIO_PID; + af->sid = SrsCodecAudioMP3; + + // for mp3, directly write to cache. + // TODO: FIXME: implements it. + for (int i = 0; i < sample->nb_sample_units; i++) { + SrsCodecSampleUnit* sample_unit = &sample->sample_units[i]; + ab->append(sample_unit->bytes, sample_unit->size); + } + } + return ret; } @@ -784,16 +872,30 @@ int SrsTsEncoder::write_audio(int64_t timestamp, char* data, int size) sample->clear(); if ((ret = codec->audio_aac_demux(data, size, sample)) != ERROR_SUCCESS) { - srs_error("http: ts codec demux audio failed. ret=%d", ret); + if (ret != ERROR_HLS_TRY_MP3) { + srs_error("http: ts aac demux audio failed. ret=%d", ret); + return ret; + } + if ((ret = codec->audio_mp3_demux(data, size, sample)) != ERROR_SUCCESS) { + srs_error("http: ts mp3 demux audio failed. ret=%d", ret); + return ret; + } + } + SrsCodecAudio acodec = (SrsCodecAudio)codec->audio_codec_id; + + // ts support audio codec: aac/mp3 + if (acodec != SrsCodecAudioAAC && acodec != SrsCodecAudioMP3) { + return ret; + } + + // when codec changed, write new header. + if ((ret = muxer->update_acodec(acodec)) != ERROR_SUCCESS) { + srs_error("http: ts audio write header failed. ret=%d", ret); return ret; } - if (codec->audio_codec_id != SrsCodecAudioAAC) { - return ret; - } - - // ignore sequence header - if (sample->aac_packet_type == SrsCodecAudioTypeSequenceHeader) { + // for aac: ignore sequence header + if (acodec == SrsCodecAudioAAC && sample->aac_packet_type == SrsCodecAudioTypeSequenceHeader) { return ret; } @@ -809,12 +911,15 @@ int SrsTsEncoder::write_audio(int64_t timestamp, char* data, int size) // flush if buffer exceed max size. if (cache->ab->length() > SRS_AUTO_HLS_AUDIO_CACHE_SIZE) { - if ((ret = muxer->write_audio(cache->af, cache->ab)) != ERROR_SUCCESS) { - return ret; - } - - // write success, clear and free the buffer - cache->ab->erase(cache->ab->length()); + return flush_video(); + } + + // TODO: config it. + // in ms, audio delay to flush the audios. + int64_t audio_delay = SRS_CONF_DEFAULT_AAC_DELAY; + // flush if audio delay exceed + if (dts - cache->audio_buffer_start_pts > audio_delay * 90) { + return flush_audio(); } return ret; @@ -852,6 +957,27 @@ int SrsTsEncoder::write_video(int64_t timestamp, char* data, int size) if ((ret = cache->cache_video(codec, dts, sample)) != ERROR_SUCCESS) { return ret; } + + return flush_video(); +} + +int SrsTsEncoder::flush_audio() +{ + int ret = ERROR_SUCCESS; + + if ((ret = muxer->write_audio(cache->af, cache->ab)) != ERROR_SUCCESS) { + return ret; + } + + // write success, clear and free the buffer + cache->ab->erase(cache->ab->length()); + + return ret; +} + +int SrsTsEncoder::flush_video() +{ + int ret = ERROR_SUCCESS; if ((ret = muxer->write_video(cache->vf, cache->vb)) != ERROR_SUCCESS) { return ret; diff --git a/trunk/src/kernel/srs_kernel_ts.hpp b/trunk/src/kernel/srs_kernel_ts.hpp index 4ee411924..3f534b171 100644 --- a/trunk/src/kernel/srs_kernel_ts.hpp +++ b/trunk/src/kernel/srs_kernel_ts.hpp @@ -31,6 +31,8 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include +#include + class SrsTsCache; class SrsTSMuxer; class SrsFileWriter; @@ -62,6 +64,9 @@ public: */ class SrsTSMuxer { +private: + SrsCodecAudio previous; + SrsCodecAudio current; private: SrsFileWriter* writer; std::string path; @@ -69,9 +74,31 @@ public: SrsTSMuxer(SrsFileWriter* w); virtual ~SrsTSMuxer(); public: + /** + * open the writer, donot write the PSI of ts. + */ virtual int open(std::string _path); + /** + * when open ts, we donot write the header(PSI), + * for user may need to update the acodec to mp3 or others, + * so we use delay write PSI, when write audio or video. + * @remark for audio aac codec, for example, SRS1, it's ok to write PSI when open ts. + * @see https://github.com/winlinvip/simple-rtmp-server/issues/301 + */ + virtual int update_acodec(SrsCodecAudio acodec); + /** + * write an audio frame to ts, + * @remark write PSI first when not write yet. + */ virtual int write_audio(SrsMpegtsFrame* af, SrsSimpleBuffer* ab); + /** + * write a video frame to ts, + * @remark write PSI first when not write yet. + */ virtual int write_video(SrsMpegtsFrame* vf, SrsSimpleBuffer* vb); + /** + * close the writer. + */ virtual void close(); }; @@ -125,6 +152,10 @@ public: SrsSimpleBuffer* ab; SrsMpegtsFrame* vf; SrsSimpleBuffer* vb; +public: + // the audio cache buffer start pts, to flush audio if full. + // @remark the pts is not the adjust one, it's the orignal pts. + int64_t audio_buffer_start_pts; protected: // time jitter for aac SrsTsAacJitter* aac_jitter; @@ -172,6 +203,9 @@ public: */ virtual int write_audio(int64_t timestamp, char* data, int size); virtual int write_video(int64_t timestamp, char* data, int size); +private: + virtual int flush_audio(); + virtual int flush_video(); }; #endif