for #304, rewrite annexb mux for ts, refer to apple sample. 2.0.157.

README.md

@@ -562,6 +562,7 @@ Supported operating systems and hardware:
 
 ### SRS 2.0 history
 
+* v2.0, 2015-04-04, for [#304](https://github.com/winlinvip/simple-rtmp-server/issues/304), rewrite annexb mux for ts, refer to apple sample. 2.0.157.
 * v2.0, 2015-04-03, enhanced avc decode, parse the sps get width+height. 2.0.156.
 * v2.0, 2015-04-03, for [#372](https://github.com/winlinvip/simple-rtmp-server/issues/372), support transform vhost of edge 2.0.155.
 * v2.0, 2015-03-30, for [#366](https://github.com/winlinvip/simple-rtmp-server/issues/366), config hls to disable cleanup of ts. 2.0.154.

trunk/src/app/srs_app_hls.cpp

@@ -754,7 +754,7 @@ int SrsHlsMuxer::_refresh_m3u8(string m3u8_file)
         // "#EXTINF:4294967295.208,\n"
         ss.precision(3);
         ss.setf(std::ios::fixed, std::ios::floatfield);
-        ss << "#EXTINF:" << segment->duration << "," << SRS_CONSTS_LF;
+        ss << "#EXTINF:" << segment->duration << ", no desc" << SRS_CONSTS_LF;
         srs_verbose("write m3u8 segment info success.");
         
         // {file name}\n
@@ -918,9 +918,12 @@ int SrsHlsCache::write_video(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t
     }
     
     // new segment when:
-    // 1. base on gop.
+    // 1. base on gop(IDR).
     // 2. some gops duration overflow.
     if (sample->frame_type == SrsCodecVideoAVCFrameKeyFrame && muxer->is_segment_overflow()) {
+        if (!sample->has_idr) {
+            srs_warn("hls: ts starts without IDR, first nalu=%d", sample->first_nalu_type);
+        }
         if ((ret = reap_segment("video", muxer, cache->video->dts)) != ERROR_SUCCESS) {
             return ret;
         }

trunk/src/core/srs_core.hpp

@@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 // current release version
 #define VERSION_MAJOR       2
 #define VERSION_MINOR       0
-#define VERSION_REVISION    156
+#define VERSION_REVISION    157
 
 // server info.
 #define RTMP_SIG_SRS_KEY "SRS"

trunk/src/kernel/srs_kernel_codec.cpp

@@ -266,6 +266,30 @@ bool SrsFlvCodec::audio_is_aac(char* data, int size)
     return sound_format == SrsCodecAudioAAC;
 }
 
+string srs_codec_avc_nalu2str(SrsAvcNaluType nalu_type)
+{
+    switch (nalu_type) {
+        case SrsAvcNaluTypeNonIDR: return "NonIDR";
+        case SrsAvcNaluTypeDataPartitionA: return "DataPartitionA";
+        case SrsAvcNaluTypeDataPartitionB: return "DataPartitionB";
+        case SrsAvcNaluTypeDataPartitionC: return "DataPartitionC";
+        case SrsAvcNaluTypeIDR: return "IDR";
+        case SrsAvcNaluTypeSEI: return "SEI";
+        case SrsAvcNaluTypeSPS: return "SPS";
+        case SrsAvcNaluTypePPS: return "PPS";
+        case SrsAvcNaluTypeAccessUnitDelimiter: return "AccessUnitDelimiter";
+        case SrsAvcNaluTypeEOSequence: return "EOSequence";
+        case SrsAvcNaluTypeEOStream: return "EOStream";
+        case SrsAvcNaluTypeFilterData: return "FilterData";
+        case SrsAvcNaluTypeSPSExt: return "SPSExt";
+        case SrsAvcNaluTypePrefixNALU: return "PrefixNALU";
+        case SrsAvcNaluTypeSubsetSPS: return "SubsetSPS";
+        case SrsAvcNaluTypeLayerWithoutPartition: return "LayerWithoutPartition";
+        case SrsAvcNaluTypeCodedSliceExt: return "CodedSliceExt";
+        case SrsAvcNaluTypeReserved: default: return "Other";
+    }
+}
+
 SrsCodecSampleUnit::SrsCodecSampleUnit()
 {
     size = 0;
@@ -293,6 +317,8 @@ void SrsCodecSample::clear()
     cts = 0;
     frame_type = SrsCodecVideoAVCFrameReserved;
     avc_packet_type = SrsCodecVideoAVCTypeReserved;
+    has_idr = false;
+    first_nalu_type = SrsAvcNaluTypeReserved;
     
     acodec = SrsCodecAudioReserved1;
     sound_rate = SrsCodecAudioSampleRateReserved;
@@ -316,6 +342,19 @@ int SrsCodecSample::add_sample_unit(char* bytes, int size)
     sample_unit->bytes = bytes;
     sample_unit->size = size;
     
+    // for video, parse the nalu type, set the IDR flag.
+    if (is_video) {
+        SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(bytes[0] & 0x1f);
+        
+        if (nal_unit_type == SrsAvcNaluTypeIDR) {
+            has_idr = true;
+        }
+    
+        if (first_nalu_type == SrsAvcNaluTypeReserved) {
+            first_nalu_type = nal_unit_type;
+        }
+    }
+    
     return ret;
 }
 
@@ -791,7 +830,7 @@ int SrsAvcAacCodec::avc_demux_sps()
     }
     
     // for NALU, 7.3.1 NAL unit syntax
-    // H.264-AVC-ISO_IEC_14496-10.pdf, page 44.
+    // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 61.
     if (!stream.require(1)) {
         ret = ERROR_HLS_DECODE_ERROR;
         srs_error("avc decode sps failed. ret=%d", ret);
@@ -816,10 +855,10 @@ int SrsAvcAacCodec::avc_demux_sps()
         return ret;
     }
     
+    // 7.4.1 NAL unit semantics
+    // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 61.
     // nal_unit_type specifies the type of RBSP data structure contained in the NAL unit as specified in Table 7-1.
-    // VCL NAL units are specified as those NAL units having nal_unit_type equal to 1, 2, 3, 4, 5, or 12.
+    SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(nutv & 0x1f);
-    // All remaining NAL units are called non-VCL NAL units.
-    int8_t nal_unit_type = nutv & 0x1f;
     if (nal_unit_type != 7) {
         ret = ERROR_HLS_DECODE_ERROR;
         srs_error("for sps, nal_unit_type shall be equal to 7. ret=%d", ret);

trunk/src/kernel/srs_kernel_codec.hpp

@@ -275,6 +275,52 @@ enum SrsCodecAudioSoundType
     SrsCodecAudioSoundTypeStereo                    = 1,
 };
 
+/**
+ * Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes
+ * H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 83.
+ */
+enum SrsAvcNaluType
+{
+    // Unspecified
+    SrsAvcNaluTypeReserved = 0,
+    
+    // Coded slice of a non-IDR picture slice_layer_without_partitioning_rbsp( )
+    SrsAvcNaluTypeNonIDR = 1,
+    // Coded slice data partition A slice_data_partition_a_layer_rbsp( )
+    SrsAvcNaluTypeDataPartitionA = 2,
+    // Coded slice data partition B slice_data_partition_b_layer_rbsp( )
+    SrsAvcNaluTypeDataPartitionB = 3,
+    // Coded slice data partition C slice_data_partition_c_layer_rbsp( )
+    SrsAvcNaluTypeDataPartitionC = 4,
+    // Coded slice of an IDR picture slice_layer_without_partitioning_rbsp( )
+    SrsAvcNaluTypeIDR = 5,
+    // Supplemental enhancement information (SEI) sei_rbsp( )
+    SrsAvcNaluTypeSEI = 6,
+    // Sequence parameter set seq_parameter_set_rbsp( )
+    SrsAvcNaluTypeSPS = 7,
+    // Picture parameter set pic_parameter_set_rbsp( )
+    SrsAvcNaluTypePPS = 8,
+    // Access unit delimiter access_unit_delimiter_rbsp( )
+    SrsAvcNaluTypeAccessUnitDelimiter = 9,
+    // End of sequence end_of_seq_rbsp( )
+    SrsAvcNaluTypeEOSequence = 10,
+    // End of stream end_of_stream_rbsp( )
+    SrsAvcNaluTypeEOStream = 11,
+    // Filler data filler_data_rbsp( )
+    SrsAvcNaluTypeFilterData = 12,
+    // Sequence parameter set extension seq_parameter_set_extension_rbsp( )
+    SrsAvcNaluTypeSPSExt = 13,
+    // Prefix NAL unit prefix_nal_unit_rbsp( )
+    SrsAvcNaluTypePrefixNALU = 14,
+    // Subset sequence parameter set subset_seq_parameter_set_rbsp( )
+    SrsAvcNaluTypeSubsetSPS = 15,
+    // Coded slice of an auxiliary coded picture without partitioning slice_layer_without_partitioning_rbsp( )
+    SrsAvcNaluTypeLayerWithoutPartition = 19,
+    // Coded slice extension slice_layer_extension_rbsp( )
+    SrsAvcNaluTypeCodedSliceExt = 20,
+};
+std::string srs_codec_avc_nalu2str(SrsAvcNaluType nalu_type);
+
 /**
 * the codec sample unit.
 * for h.264 video packet, a NALU is a sample unit.
@@ -334,6 +380,9 @@ public:
     // video specified
     SrsCodecVideoAVCFrame frame_type;
     SrsCodecVideoAVCType avc_packet_type;
+    // whether sample_units contains IDR frame.
+    bool has_idr;
+    SrsAvcNaluType first_nalu_type;
 public:
     // audio specified
     SrsCodecAudio acodec;

trunk/src/kernel/srs_kernel_ts.cpp

@@ -2835,19 +2835,104 @@ int SrsTsCache::do_cache_avc(SrsAvcAacCodec* codec, SrsCodecSample* sample)
 {
     int ret = ERROR_SUCCESS;
     
-    // for type1/5/6, insert aud packet.
+    // mux the samples in annexb format,
-    u_int8_t aud_nal[] = { 0x00, 0x00, 0x00, 0x01, 0x09, 0xf0 };
+    // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 324.
-    
-    bool sps_pps_sent = false;
-    bool aud_sent = false;
     /**
-    * a ts sample is format as:
      * 00 00 00 01 // header
      *       xxxxxxx // data bytes
      * 00 00 01 // continue header
      *       xxxxxxx // data bytes.
-    * so, for each sample, we append header in aud_nal, then appends the bytes in sample.
+     *
+     * nal_unit_type specifies the type of RBSP data structure contained in the NAL unit as specified in Table 7-1.
+     * Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes
+     * H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 83.
+     *      1, Coded slice of a non-IDR picture slice_layer_without_partitioning_rbsp( )
+     *      2, Coded slice data partition A slice_data_partition_a_layer_rbsp( )
+     *      3, Coded slice data partition B slice_data_partition_b_layer_rbsp( )
+     *      4, Coded slice data partition C slice_data_partition_c_layer_rbsp( )
+     *      5, Coded slice of an IDR picture slice_layer_without_partitioning_rbsp( )
+     *      6, Supplemental enhancement information (SEI) sei_rbsp( )
+     *      7, Sequence parameter set seq_parameter_set_rbsp( )
+     *      8, Picture parameter set pic_parameter_set_rbsp( )
+     *      9, Access unit delimiter access_unit_delimiter_rbsp( )
+     *      10, End of sequence end_of_seq_rbsp( )
+     *      11, End of stream end_of_stream_rbsp( )
+     *      12, Filler data filler_data_rbsp( )
+     *      13, Sequence parameter set extension seq_parameter_set_extension_rbsp( )
+     *      14, Prefix NAL unit prefix_nal_unit_rbsp( )
+     *      15, Subset sequence parameter set subset_seq_parameter_set_rbsp( )
+     *      19, Coded slice of an auxiliary coded picture without partitioning slice_layer_without_partitioning_rbsp( )
+     *      20, Coded slice extension slice_layer_extension_rbsp( )
+     * the first ts message of apple sample:
+     *      annexb 4B header, 2B aud(nal_unit_type:6)(0x09 0xf0)
+     *      annexb 4B header, 19B sps(nal_unit_type:7)
+     *      annexb 3B header, 4B pps(nal_unit_type:8)
+     *      annexb 3B header, 12B nalu(nal_unit_type:6)
+     *      annexb 3B header, 21B nalu(nal_unit_type:6)
+     *      annexb 3B header, 2762B nalu(nal_unit_type:5)
+     *      annexb 3B header, 3535B nalu(nal_unit_type:5)
+     * the second ts message of apple ts sample:
+     *      annexb 4B header, 2B aud(nal_unit_type:6)(0x09 0xf0)
+     *      annexb 3B header, 21B nalu(nal_unit_type:6)
+     *      annexb 3B header, 379B nalu(nal_unit_type:1)
+     *      annexb 3B header, 406B nalu(nal_unit_type:1)
      */
+    static u_int8_t fresh_nalu_header[] = { 0x00, 0x00, 0x00, 0x01 };
+    static u_int8_t cont_nalu_header[] = { 0x00, 0x00, 0x01 };
+    
+    // the aud(access unit delimiter) before each frame.
+    // 7.3.2.4 Access unit delimiter RBSP syntax
+    // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 66.
+    //
+    // primary_pic_type u(3), the first 3bits, primary_pic_type indicates that the slice_type values
+    //      for all slices of the primary coded picture are members of the set listed in Table 7-5 for
+    //      the given value of primary_pic_type.
+    //      0, slice_type 2, 7
+    //      1, slice_type 0, 2, 5, 7
+    //      2, slice_type 0, 1, 2, 5, 6, 7
+    //      3, slice_type 4, 9
+    //      4, slice_type 3, 4, 8, 9
+    //      5, slice_type 2, 4, 7, 9
+    //      6, slice_type 0, 2, 3, 4, 5, 7, 8, 9
+    //      7, slice_type 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // 7.4.2.4 Access unit delimiter RBSP semantics
+    // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 102.
+    //
+    // slice_type specifies the coding type of the slice according to Table 7-6.
+    //      0, P (P slice)
+    //      1, B (B slice)
+    //      2, I (I slice)
+    //      3, SP (SP slice)
+    //      4, SI (SI slice)
+    //      5, P (P slice)
+    //      6, B (B slice)
+    //      7, I (I slice)
+    //      8, SP (SP slice)
+    //      9, SI (SI slice)
+    // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 105.
+    static u_int8_t aud_nalu_7[] = { 0x09, 0xf0};
+    video->payload->append((const char*)fresh_nalu_header, 4);
+    video->payload->append((const char*)aud_nalu_7, 2);
+    
+    // when ts message(samples) contains IDR, insert sps+pps.
+    if (sample->has_idr) {
+        // fresh nalu header before sps.
+        if (codec->sequenceParameterSetLength > 0) {
+            // AnnexB prefix, for sps always 4 bytes header
+            video->payload->append((const char*)fresh_nalu_header, 4);
+            // sps
+            video->payload->append(codec->sequenceParameterSetNALUnit, codec->sequenceParameterSetLength);
+        }
+        // cont nalu header before pps.
+        if (codec->pictureParameterSetLength > 0) {
+            // AnnexB prefix, for pps always 3 bytes header
+            video->payload->append((const char*)cont_nalu_header, 3);
+            // pps
+            video->payload->append(codec->pictureParameterSetNALUnit, codec->pictureParameterSetLength);
+        }
+    }
+    
+    // all sample use cont nalu header, except the sps-pps before IDR frame.
     for (int i = 0; i < sample->nb_sample_units; i++) {
         SrsCodecSampleUnit* sample_unit = &sample->sample_units[i];
         int32_t size = sample_unit->size;
@@ -2858,83 +2943,22 @@ int SrsTsCache::do_cache_avc(SrsAvcAacCodec* codec, SrsCodecSample* sample)
             return ret;
         }
         
-        /**
-        * step 1:
-        * first, before each "real" sample, 
-        * we add some packets according to the nal_unit_type,
-        * for example, when got nal_unit_type=5, insert SPS/PPS before sample.
-        */
-        
         // 5bits, 7.3.1 NAL unit syntax,
-        // H.264-AVC-ISO_IEC_14496-10.pdf, page 44.
+        // H.264-AVC-ISO_IEC_14496-10-2012.pdf, page 83.
-        u_int8_t nal_unit_type;
+        SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(sample_unit->bytes[0] & 0x1f);
-        nal_unit_type = *sample_unit->bytes;
-        nal_unit_type &= 0x1f;
         
-        // @see: ngx_rtmp_hls_video
+        // ignore SPS/PPS/AUD
-        // Table 7-1 NAL unit type codes, page 61
+        switch (nal_unit_type) {
-        // 1: Coded slice
+            case SrsAvcNaluTypeSPS:
-        if (nal_unit_type == 1) {
+            case SrsAvcNaluTypePPS:
-            sps_pps_sent = false;
+            case SrsAvcNaluTypeAccessUnitDelimiter:
-        }
-        
-        // 6: Supplemental enhancement information (SEI) sei_rbsp( ), page 61
-        // @see: ngx_rtmp_hls_append_aud
-        if (!aud_sent) {
-            // @remark, when got type 9, we donot send aud_nal, but it will make 
-            //      ios unhappy, so we remove it.
-            // @see https://github.com/winlinvip/simple-rtmp-server/issues/281
-            /*if (nal_unit_type == 9) {
-                aud_sent = true;
-            }*/
-            
-            if (nal_unit_type == 1 || nal_unit_type == 5 || nal_unit_type == 6) {
-                // for type 6, append a aud with type 9.
-                video->payload->append((const char*)aud_nal, sizeof(aud_nal));
-                aud_sent = true;
-            }
-        }
-        
-        // 5: Coded slice of an IDR picture.
-        // insert sps/pps before IDR or key frame is ok.
-        if (nal_unit_type == 5 && !sps_pps_sent) {
-            sps_pps_sent = true;
-            
-            // @see: ngx_rtmp_hls_append_sps_pps
-            if (codec->sequenceParameterSetLength > 0) {
-                // AnnexB prefix, for sps always 4 bytes header
-                video->payload->append((const char*)aud_nal, 4);
-                // sps
-                video->payload->append(codec->sequenceParameterSetNALUnit, codec->sequenceParameterSetLength);
-            }
-            if (codec->pictureParameterSetLength > 0) {
-                // AnnexB prefix, for pps always 4 bytes header
-                video->payload->append((const char*)aud_nal, 4);
-                // pps
-                video->payload->append(codec->pictureParameterSetNALUnit, codec->pictureParameterSetLength);
-            }
-        }
-        
-        // 7-9, ignore, @see: ngx_rtmp_hls_video
-        if (nal_unit_type >= 7 && nal_unit_type <= 9) {
                 continue;
+            default:
+                break;
         }
         
-        /**
+        // insert cont nalu header before frame.
-        * step 2:
+        video->payload->append((const char*)cont_nalu_header, 3);
-        * output the "real" sample, in buf.
-        * when we output some special assist packets according to nal_unit_type
-        */
-        
-        // sample start prefix, '00 00 00 01' or '00 00 01'
-        u_int8_t* p = aud_nal + 1;
-        u_int8_t* end = p + 3;
-        
-        // first AnnexB prefix is long (4 bytes)
-        if (video->payload->length() == 0) {
-            p = aud_nal;
-        }
-        video->payload->append((const char*)p, end - p);
         
         // sample data
         video->payload->append(sample_unit->bytes, sample_unit->size);