From 77c29da26b5f435b7f04d4158c317f4dea4b0f02 Mon Sep 17 00:00:00 2001 From: winlin Date: Sat, 4 Apr 2015 13:30:13 +0800 Subject: [PATCH] for #304, rewrite annexb mux for ts, refer to apple sample. 2.0.157. --- README.md | 1 + trunk/src/app/srs_app_hls.cpp | 7 +- trunk/src/core/srs_core.hpp | 2 +- trunk/src/kernel/srs_kernel_codec.cpp | 47 +++++- trunk/src/kernel/srs_kernel_codec.hpp | 49 +++++++ trunk/src/kernel/srs_kernel_ts.cpp | 196 +++++++++++++++----------- 6 files changed, 209 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index dac38a66e..d1ededdc0 100755 --- a/README.md +++ b/README.md @@ -562,6 +562,7 @@ Supported operating systems and hardware: ### SRS 2.0 history +* v2.0, 2015-04-04, for [#304](https://github.com/winlinvip/simple-rtmp-server/issues/304), rewrite annexb mux for ts, refer to apple sample. 2.0.157. * v2.0, 2015-04-03, enhanced avc decode, parse the sps get width+height. 2.0.156. * v2.0, 2015-04-03, for [#372](https://github.com/winlinvip/simple-rtmp-server/issues/372), support transform vhost of edge 2.0.155. * v2.0, 2015-03-30, for [#366](https://github.com/winlinvip/simple-rtmp-server/issues/366), config hls to disable cleanup of ts. 2.0.154. diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index f9a329fdb..57e77153a 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -754,7 +754,7 @@ int SrsHlsMuxer::_refresh_m3u8(string m3u8_file) // "#EXTINF:4294967295.208,\n" ss.precision(3); ss.setf(std::ios::fixed, std::ios::floatfield); - ss << "#EXTINF:" << segment->duration << "," << SRS_CONSTS_LF; + ss << "#EXTINF:" << segment->duration << ", no desc" << SRS_CONSTS_LF; srs_verbose("write m3u8 segment info success."); // {file name}\n @@ -918,9 +918,12 @@ int SrsHlsCache::write_video(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t } // new segment when: - // 1. base on gop. + // 1. base on gop(IDR). // 2. some gops duration overflow. if (sample->frame_type == SrsCodecVideoAVCFrameKeyFrame && muxer->is_segment_overflow()) { + if (!sample->has_idr) { + srs_warn("hls: ts starts without IDR, first nalu=%d", sample->first_nalu_type); + } if ((ret = reap_segment("video", muxer, cache->video->dts)) != ERROR_SUCCESS) { return ret; } diff --git a/trunk/src/core/srs_core.hpp b/trunk/src/core/srs_core.hpp index fc6dc8140..5d93659d0 100644 --- a/trunk/src/core/srs_core.hpp +++ b/trunk/src/core/srs_core.hpp @@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // current release version #define VERSION_MAJOR 2 #define VERSION_MINOR 0 -#define VERSION_REVISION 156 +#define VERSION_REVISION 157 // server info. #define RTMP_SIG_SRS_KEY "SRS" diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 78299692f..640b4db1a 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -266,6 +266,30 @@ bool SrsFlvCodec::audio_is_aac(char* data, int size) return sound_format == SrsCodecAudioAAC; } +string srs_codec_avc_nalu2str(SrsAvcNaluType nalu_type) +{ + switch (nalu_type) { + case SrsAvcNaluTypeNonIDR: return "NonIDR"; + case SrsAvcNaluTypeDataPartitionA: return "DataPartitionA"; + case SrsAvcNaluTypeDataPartitionB: return "DataPartitionB"; + case SrsAvcNaluTypeDataPartitionC: return "DataPartitionC"; + case SrsAvcNaluTypeIDR: return "IDR"; + case SrsAvcNaluTypeSEI: return "SEI"; + case SrsAvcNaluTypeSPS: return "SPS"; + case SrsAvcNaluTypePPS: return "PPS"; + case SrsAvcNaluTypeAccessUnitDelimiter: return "AccessUnitDelimiter"; + case SrsAvcNaluTypeEOSequence: return "EOSequence"; + case SrsAvcNaluTypeEOStream: return "EOStream"; + case SrsAvcNaluTypeFilterData: return "FilterData"; + case SrsAvcNaluTypeSPSExt: return "SPSExt"; + case SrsAvcNaluTypePrefixNALU: return "PrefixNALU"; + case SrsAvcNaluTypeSubsetSPS: return "SubsetSPS"; + case SrsAvcNaluTypeLayerWithoutPartition: return "LayerWithoutPartition"; + case SrsAvcNaluTypeCodedSliceExt: return "CodedSliceExt"; + case SrsAvcNaluTypeReserved: default: return "Other"; + } +} + SrsCodecSampleUnit::SrsCodecSampleUnit() { size = 0; @@ -293,6 +317,8 @@ void SrsCodecSample::clear() cts = 0; frame_type = SrsCodecVideoAVCFrameReserved; avc_packet_type = SrsCodecVideoAVCTypeReserved; + has_idr = false; + first_nalu_type = SrsAvcNaluTypeReserved; acodec = SrsCodecAudioReserved1; sound_rate = SrsCodecAudioSampleRateReserved; @@ -316,6 +342,19 @@ int SrsCodecSample::add_sample_unit(char* bytes, int size) sample_unit->bytes = bytes; sample_unit->size = size; + // for video, parse the nalu type, set the IDR flag. + if (is_video) { + SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(bytes[0] & 0x1f); + + if (nal_unit_type == SrsAvcNaluTypeIDR) { + has_idr = true; + } + + if (first_nalu_type == SrsAvcNaluTypeReserved) { + first_nalu_type = nal_unit_type; + } + } + return ret; } @@ -791,7 +830,7 @@ int SrsAvcAacCodec::avc_demux_sps() } // for NALU, 7.3.1 NAL unit syntax - // H.264-AVC-ISO_IEC_14496-10.pdf, page 44. + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 61. if (!stream.require(1)) { ret = ERROR_HLS_DECODE_ERROR; srs_error("avc decode sps failed. ret=%d", ret); @@ -816,10 +855,10 @@ int SrsAvcAacCodec::avc_demux_sps() return ret; } + // 7.4.1 NAL unit semantics + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 61. // nal_unit_type specifies the type of RBSP data structure contained in the NAL unit as specified in Table 7-1. - // VCL NAL units are specified as those NAL units having nal_unit_type equal to 1, 2, 3, 4, 5, or 12. - // All remaining NAL units are called non-VCL NAL units. - int8_t nal_unit_type = nutv & 0x1f; + SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(nutv & 0x1f); if (nal_unit_type != 7) { ret = ERROR_HLS_DECODE_ERROR; srs_error("for sps, nal_unit_type shall be equal to 7. ret=%d", ret); diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index ce174dc3a..4df7596bd 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -275,6 +275,52 @@ enum SrsCodecAudioSoundType SrsCodecAudioSoundTypeStereo = 1, }; +/** + * Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes + * H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 83. + */ +enum SrsAvcNaluType +{ + // Unspecified + SrsAvcNaluTypeReserved = 0, + + // Coded slice of a non-IDR picture slice_layer_without_partitioning_rbsp( ) + SrsAvcNaluTypeNonIDR = 1, + // Coded slice data partition A slice_data_partition_a_layer_rbsp( ) + SrsAvcNaluTypeDataPartitionA = 2, + // Coded slice data partition B slice_data_partition_b_layer_rbsp( ) + SrsAvcNaluTypeDataPartitionB = 3, + // Coded slice data partition C slice_data_partition_c_layer_rbsp( ) + SrsAvcNaluTypeDataPartitionC = 4, + // Coded slice of an IDR picture slice_layer_without_partitioning_rbsp( ) + SrsAvcNaluTypeIDR = 5, + // Supplemental enhancement information (SEI) sei_rbsp( ) + SrsAvcNaluTypeSEI = 6, + // Sequence parameter set seq_parameter_set_rbsp( ) + SrsAvcNaluTypeSPS = 7, + // Picture parameter set pic_parameter_set_rbsp( ) + SrsAvcNaluTypePPS = 8, + // Access unit delimiter access_unit_delimiter_rbsp( ) + SrsAvcNaluTypeAccessUnitDelimiter = 9, + // End of sequence end_of_seq_rbsp( ) + SrsAvcNaluTypeEOSequence = 10, + // End of stream end_of_stream_rbsp( ) + SrsAvcNaluTypeEOStream = 11, + // Filler data filler_data_rbsp( ) + SrsAvcNaluTypeFilterData = 12, + // Sequence parameter set extension seq_parameter_set_extension_rbsp( ) + SrsAvcNaluTypeSPSExt = 13, + // Prefix NAL unit prefix_nal_unit_rbsp( ) + SrsAvcNaluTypePrefixNALU = 14, + // Subset sequence parameter set subset_seq_parameter_set_rbsp( ) + SrsAvcNaluTypeSubsetSPS = 15, + // Coded slice of an auxiliary coded picture without partitioning slice_layer_without_partitioning_rbsp( ) + SrsAvcNaluTypeLayerWithoutPartition = 19, + // Coded slice extension slice_layer_extension_rbsp( ) + SrsAvcNaluTypeCodedSliceExt = 20, +}; +std::string srs_codec_avc_nalu2str(SrsAvcNaluType nalu_type); + /** * the codec sample unit. * for h.264 video packet, a NALU is a sample unit. @@ -334,6 +380,9 @@ public: // video specified SrsCodecVideoAVCFrame frame_type; SrsCodecVideoAVCType avc_packet_type; + // whether sample_units contains IDR frame. + bool has_idr; + SrsAvcNaluType first_nalu_type; public: // audio specified SrsCodecAudio acodec; diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 856482a24..b813710ad 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -2835,19 +2835,104 @@ int SrsTsCache::do_cache_avc(SrsAvcAacCodec* codec, SrsCodecSample* sample) { int ret = ERROR_SUCCESS; - // for type1/5/6, insert aud packet. - u_int8_t aud_nal[] = { 0x00, 0x00, 0x00, 0x01, 0x09, 0xf0 }; - - bool sps_pps_sent = false; - bool aud_sent = false; + // mux the samples in annexb format, + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 324. /** - * a ts sample is format as: - * 00 00 00 01 // header - * xxxxxxx // data bytes - * 00 00 01 // continue header - * xxxxxxx // data bytes. - * so, for each sample, we append header in aud_nal, then appends the bytes in sample. - */ + * 00 00 00 01 // header + * xxxxxxx // data bytes + * 00 00 01 // continue header + * xxxxxxx // data bytes. + * + * nal_unit_type specifies the type of RBSP data structure contained in the NAL unit as specified in Table 7-1. + * Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes + * H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 83. + * 1, Coded slice of a non-IDR picture slice_layer_without_partitioning_rbsp( ) + * 2, Coded slice data partition A slice_data_partition_a_layer_rbsp( ) + * 3, Coded slice data partition B slice_data_partition_b_layer_rbsp( ) + * 4, Coded slice data partition C slice_data_partition_c_layer_rbsp( ) + * 5, Coded slice of an IDR picture slice_layer_without_partitioning_rbsp( ) + * 6, Supplemental enhancement information (SEI) sei_rbsp( ) + * 7, Sequence parameter set seq_parameter_set_rbsp( ) + * 8, Picture parameter set pic_parameter_set_rbsp( ) + * 9, Access unit delimiter access_unit_delimiter_rbsp( ) + * 10, End of sequence end_of_seq_rbsp( ) + * 11, End of stream end_of_stream_rbsp( ) + * 12, Filler data filler_data_rbsp( ) + * 13, Sequence parameter set extension seq_parameter_set_extension_rbsp( ) + * 14, Prefix NAL unit prefix_nal_unit_rbsp( ) + * 15, Subset sequence parameter set subset_seq_parameter_set_rbsp( ) + * 19, Coded slice of an auxiliary coded picture without partitioning slice_layer_without_partitioning_rbsp( ) + * 20, Coded slice extension slice_layer_extension_rbsp( ) + * the first ts message of apple sample: + * annexb 4B header, 2B aud(nal_unit_type:6)(0x09 0xf0) + * annexb 4B header, 19B sps(nal_unit_type:7) + * annexb 3B header, 4B pps(nal_unit_type:8) + * annexb 3B header, 12B nalu(nal_unit_type:6) + * annexb 3B header, 21B nalu(nal_unit_type:6) + * annexb 3B header, 2762B nalu(nal_unit_type:5) + * annexb 3B header, 3535B nalu(nal_unit_type:5) + * the second ts message of apple ts sample: + * annexb 4B header, 2B aud(nal_unit_type:6)(0x09 0xf0) + * annexb 3B header, 21B nalu(nal_unit_type:6) + * annexb 3B header, 379B nalu(nal_unit_type:1) + * annexb 3B header, 406B nalu(nal_unit_type:1) + */ + static u_int8_t fresh_nalu_header[] = { 0x00, 0x00, 0x00, 0x01 }; + static u_int8_t cont_nalu_header[] = { 0x00, 0x00, 0x01 }; + + // the aud(access unit delimiter) before each frame. + // 7.3.2.4 Access unit delimiter RBSP syntax + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 66. + // + // primary_pic_type u(3), the first 3bits, primary_pic_type indicates that the slice_type values + // for all slices of the primary coded picture are members of the set listed in Table 7-5 for + // the given value of primary_pic_type. + // 0, slice_type 2, 7 + // 1, slice_type 0, 2, 5, 7 + // 2, slice_type 0, 1, 2, 5, 6, 7 + // 3, slice_type 4, 9 + // 4, slice_type 3, 4, 8, 9 + // 5, slice_type 2, 4, 7, 9 + // 6, slice_type 0, 2, 3, 4, 5, 7, 8, 9 + // 7, slice_type 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 + // 7.4.2.4 Access unit delimiter RBSP semantics + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 102. + // + // slice_type specifies the coding type of the slice according to Table 7-6. + // 0, P (P slice) + // 1, B (B slice) + // 2, I (I slice) + // 3, SP (SP slice) + // 4, SI (SI slice) + // 5, P (P slice) + // 6, B (B slice) + // 7, I (I slice) + // 8, SP (SP slice) + // 9, SI (SI slice) + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 105. + static u_int8_t aud_nalu_7[] = { 0x09, 0xf0}; + video->payload->append((const char*)fresh_nalu_header, 4); + video->payload->append((const char*)aud_nalu_7, 2); + + // when ts message(samples) contains IDR, insert sps+pps. + if (sample->has_idr) { + // fresh nalu header before sps. + if (codec->sequenceParameterSetLength > 0) { + // AnnexB prefix, for sps always 4 bytes header + video->payload->append((const char*)fresh_nalu_header, 4); + // sps + video->payload->append(codec->sequenceParameterSetNALUnit, codec->sequenceParameterSetLength); + } + // cont nalu header before pps. + if (codec->pictureParameterSetLength > 0) { + // AnnexB prefix, for pps always 3 bytes header + video->payload->append((const char*)cont_nalu_header, 3); + // pps + video->payload->append(codec->pictureParameterSetNALUnit, codec->pictureParameterSetLength); + } + } + + // all sample use cont nalu header, except the sps-pps before IDR frame. for (int i = 0; i < sample->nb_sample_units; i++) { SrsCodecSampleUnit* sample_unit = &sample->sample_units[i]; int32_t size = sample_unit->size; @@ -2858,83 +2943,22 @@ int SrsTsCache::do_cache_avc(SrsAvcAacCodec* codec, SrsCodecSample* sample) return ret; } - /** - * step 1: - * first, before each "real" sample, - * we add some packets according to the nal_unit_type, - * for example, when got nal_unit_type=5, insert SPS/PPS before sample. - */ + // 5bits, 7.3.1 NAL unit syntax, + // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 83. + SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(sample_unit->bytes[0] & 0x1f); - // 5bits, 7.3.1 NAL unit syntax, - // H.264-AVC-ISO_IEC_14496-10.pdf, page 44. - u_int8_t nal_unit_type; - nal_unit_type = *sample_unit->bytes; - nal_unit_type &= 0x1f; - - // @see: ngx_rtmp_hls_video - // Table 7-1 NAL unit type codes, page 61 - // 1: Coded slice - if (nal_unit_type == 1) { - sps_pps_sent = false; + // ignore SPS/PPS/AUD + switch (nal_unit_type) { + case SrsAvcNaluTypeSPS: + case SrsAvcNaluTypePPS: + case SrsAvcNaluTypeAccessUnitDelimiter: + continue; + default: + break; } - // 6: Supplemental enhancement information (SEI) sei_rbsp( ), page 61 - // @see: ngx_rtmp_hls_append_aud - if (!aud_sent) { - // @remark, when got type 9, we donot send aud_nal, but it will make - // ios unhappy, so we remove it. - // @see https://github.com/winlinvip/simple-rtmp-server/issues/281 - /*if (nal_unit_type == 9) { - aud_sent = true; - }*/ - - if (nal_unit_type == 1 || nal_unit_type == 5 || nal_unit_type == 6) { - // for type 6, append a aud with type 9. - video->payload->append((const char*)aud_nal, sizeof(aud_nal)); - aud_sent = true; - } - } - - // 5: Coded slice of an IDR picture. - // insert sps/pps before IDR or key frame is ok. - if (nal_unit_type == 5 && !sps_pps_sent) { - sps_pps_sent = true; - - // @see: ngx_rtmp_hls_append_sps_pps - if (codec->sequenceParameterSetLength > 0) { - // AnnexB prefix, for sps always 4 bytes header - video->payload->append((const char*)aud_nal, 4); - // sps - video->payload->append(codec->sequenceParameterSetNALUnit, codec->sequenceParameterSetLength); - } - if (codec->pictureParameterSetLength > 0) { - // AnnexB prefix, for pps always 4 bytes header - video->payload->append((const char*)aud_nal, 4); - // pps - video->payload->append(codec->pictureParameterSetNALUnit, codec->pictureParameterSetLength); - } - } - - // 7-9, ignore, @see: ngx_rtmp_hls_video - if (nal_unit_type >= 7 && nal_unit_type <= 9) { - continue; - } - - /** - * step 2: - * output the "real" sample, in buf. - * when we output some special assist packets according to nal_unit_type - */ - - // sample start prefix, '00 00 00 01' or '00 00 01' - u_int8_t* p = aud_nal + 1; - u_int8_t* end = p + 3; - - // first AnnexB prefix is long (4 bytes) - if (video->payload->length() == 0) { - p = aud_nal; - } - video->payload->append((const char*)p, end - p); + // insert cont nalu header before frame. + video->payload->append((const char*)cont_nalu_header, 3); // sample data video->payload->append(sample_unit->bytes, sample_unit->size);