decoded audio and video. add avc file format doc

trunk/src/core/srs_core_codec.cpp

@@ -23,8 +23,11 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #include <srs_core_codec.hpp>
 
+#include <string.h>
+
 #include <srs_core_error.hpp>
 #include <srs_core_stream.hpp>
+#include <srs_core_log.hpp>
 
 SrsCodec::SrsCodec()
 {
@@ -36,10 +39,9 @@ SrsCodec::SrsCodec()
 	video_codec_id 			= 0;
 	audio_data_rate 		= 0;
 	audio_codec_id 			= 0;
-	aac_sample_rate	 		= 0;
+	sound_rate		 		= 0;
-	sample_rate		 		= 0;
+	sound_size		 		= 0;
-	sample_size		 		= 0;
+	sound_type	 			= 0;
-	audio_channels	 		= 0;
 	profile 				= 0;
 	level 					= 0;
 	avc_extra_size 			= 0;
@@ -58,44 +60,133 @@ SrsCodec::~SrsCodec()
 	srs_freep(stream);
 }
 
-int SrsCodec::parse_av_codec(bool is_video, int8_t* data, int size)
+int SrsCodec::parse_audio_codec(int8_t* data, int size)
 {
 	int ret = ERROR_SUCCESS;
 	
 	if (!data || size <= 0) {
+		srs_trace("no audio present, hls ignore it.");
 		return ret;
 	}
 	
 	if ((ret = stream->initialize((char*)data, size)) != ERROR_SUCCESS) {
 		return ret;
 	}
-	
+
-	if (is_video) {
+	// audio decode
-		if (!stream->require(1)) {
+	if (!stream->require(1)) {
-			return ret;
+		ret = ERROR_HLS_DECODE_ERROR;
-		}
+		srs_error("hls decode audio sound_format failed. ret=%d", ret);
-		
+		return ret;
-		int8_t frame_type = stream->read_1bytes();
-		int8_t codec_id = frame_type & 0x0f;
-		frame_type = (frame_type >> 4) & 0x0f;
-		
-		video_codec_id = codec_id;
-		if (codec_id != SrsCodecVideoAVC) {
-			return ret;
-		}
-		
-		if (!stream->require(4)) {
-			return ret;
-		}
-		int8_t avc_packet_type = stream->read_1bytes();
-		int32_t composition_time = stream->read_3bytes();
-		
-		// 5.2.4.1.1 Syntax
-		if (avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) {
-		}
-	} else {
 	}
 	
+	int8_t sound_format = stream->read_1bytes();
+	
+	sound_type = sound_format & 0x01;
+	sound_size = (sound_format >> 1) & 0x01;
+	sound_rate = (sound_format >> 2) & 0x01;
+	sound_format = (sound_format >> 4) & 0x0f;
+	
+	audio_codec_id = sound_format;
+	
+	// only support aac
+	if (audio_codec_id != SrsCodecAudioAAC) {
+		ret = ERROR_HLS_DECODE_ERROR;
+		srs_error("hls only support audio aac codec. ret=%d", ret);
+		return ret;
+	}
+
+	if (!stream->require(1)) {
+		ret = ERROR_HLS_DECODE_ERROR;
+		srs_error("hls decode audio aac_packet_type failed. ret=%d", ret);
+		return ret;
+	}
+	
+	int8_t aac_packet_type = stream->read_1bytes();
+	
+	if (aac_packet_type == SrsCodecAudioTypeSequenceHeader) {
+		// AudioSpecificConfig
+		// 1.6.2.1 AudioSpecificConfig, in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf, page 33.
+		aac_extra_size = size - stream->pos();
+		if (aac_extra_size > 0) {
+			srs_freepa(aac_extra_data);
+			aac_extra_data = new char[aac_extra_size];
+			memcpy(aac_extra_data, data + stream->pos(), aac_extra_size);
+		}
+	} else if (aac_packet_type == SrsCodecAudioTypeRawData) {
+		// Raw AAC frame data in UI8 []
+	} else {
+		// ignored.
+	}
+	
+	srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", 
+		sound_type, audio_codec_id, sound_size, sound_rate, sound_format, size);
+	
+	return ret;
+}
+
+int SrsCodec::parse_video_codec(int8_t* data, int size)
+{
+	int ret = ERROR_SUCCESS;
+	
+	if (!data || size <= 0) {
+		srs_trace("no video present, hls ignore it.");
+		return ret;
+	}
+	
+	if ((ret = stream->initialize((char*)data, size)) != ERROR_SUCCESS) {
+		return ret;
+	}
+
+	// video decode
+	if (!stream->require(1)) {
+		ret = ERROR_HLS_DECODE_ERROR;
+		srs_error("hls decode video frame_type failed. ret=%d", ret);
+		return ret;
+	}
+	
+	int8_t frame_type = stream->read_1bytes();
+	int8_t codec_id = frame_type & 0x0f;
+	frame_type = (frame_type >> 4) & 0x0f;
+	
+	video_codec_id = codec_id;
+	// only support h.264/avc
+	if (codec_id != SrsCodecVideoAVC) {
+		ret = ERROR_HLS_DECODE_ERROR;
+		srs_error("hls only support video h.264/avc codec. ret=%d", ret);
+		return ret;
+	}
+	
+	if (!stream->require(4)) {
+		ret = ERROR_HLS_DECODE_ERROR;
+		srs_error("hls decode video avc_packet_type failed. ret=%d", ret);
+		return ret;
+	}
+	int8_t avc_packet_type = stream->read_1bytes();
+	int32_t composition_time = stream->read_3bytes();
+	
+	// avoid warning, used it future.
+	(void)composition_time;
+	
+	if (avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) {
+		// AVCDecoderConfigurationRecord
+		// 5.2.4.1.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16
+		avc_extra_size = size - stream->pos();
+		if (avc_extra_size > 0) {
+			srs_freepa(avc_extra_data);
+			avc_extra_data = new char[avc_extra_size];
+			memcpy(avc_extra_data, data + stream->pos(), avc_extra_size);
+		}
+	} else if (avc_packet_type == SrsCodecVideoAVCTypeNALU){
+		// One or more NALUs (Full frames are required)
+		// 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 20
+	} else {
+		// ignored.
+	}
+	
+	srs_info("video decoded, type=%d, codec=%d, avc=%d, time=%d, size=%d", 
+		frame_type, video_codec_id, avc_packet_type, composition_time, size);
+	
 	return ret;
 }
 

trunk/src/core/srs_core_codec.hpp

@@ -128,6 +128,39 @@ enum SrsCodecAudioType
 	SrsCodecAudioTypeRawData 						= 1,
 };
 
+// Sampling rate. The following values are defined:
+// 0 = 5.5 kHz = 5512 Hz
+// 1 = 11 kHz = 11025 Hz
+// 2 = 22 kHz = 22050 Hz
+// 3 = 44 kHz = 44100 Hz
+enum SrsCodecAudioSampleRate
+{
+	SrsCodecAudioSampleRate5512 				= 0,
+	SrsCodecAudioSampleRate11025 				= 1,
+	SrsCodecAudioSampleRate22050 				= 2,
+	SrsCodecAudioSampleRate44100 				= 3,
+};
+
+// Size of each audio sample. This parameter only pertains to
+// uncompressed formats. Compressed formats always decode
+// to 16 bits internally.
+// 0 = 8-bit samples
+// 1 = 16-bit samples
+enum SrsCodecAudioSampleSize
+{
+	SrsCodecAudioSampleSize8bit 				= 0,
+	SrsCodecAudioSampleSize16bit 				= 1,
+};
+
+// Mono or stereo sound
+// 0 = Mono sound
+// 1 = Stereo sound
+enum SrsCodecAudioSoundType
+{
+	SrsCodecAudioSoundTypeMono 					= 0,
+	SrsCodecAudioSoundTypeStereo 				= 1,
+};
+
 /**
 * Annex E. The FLV File Format
 */
@@ -139,30 +172,35 @@ public:
 	/**
 	* video specified
 	*/
-	int			width;
-	int			height;
-	int			duration;
-	int			frame_rate;
 	// @see: SrsCodecVideo
 	int			video_codec_id;
-	int			video_data_rate; // in bps
 	u_int8_t	profile; // profile_idc, page 45.
 	u_int8_t	level; // level_idc, page 45.
+	int			width;
+	int			height;
+	int			video_data_rate; // in bps
+	int			frame_rate;
+	int			duration;
 	/**
 	* audio specified
 	*/
+	// @see: SrsCodecAudioType
 	int			audio_codec_id;
+	// @see: SrsCodecAudioSampleRate
+	int			sound_rate;
+	// @see: SrsCodecAudioSampleSize
+	int			sound_size;
+	// @see: SrsCodecAudioSoundType
+	int			sound_type;
 	int			audio_data_rate; // in bps
-	int			aac_sample_rate;
-	int			sample_rate; /* 5512, 11025, 22050, 44100 */
-	int			sample_size; /* 1=8bit, 2=16bit */
-	int			audio_channels; /* 1, 2 */
 	// the avc extra data, the AVC sequence header,
-	// without the flv codec header
+	// without the flv codec header,
+	// @see: ffmpeg, AVCodecContext::extradata
 	int 		avc_extra_size;
 	char*		avc_extra_data;
 	// the aac extra data, the AAC sequence header,
-	// without the flv codec header
+	// without the flv codec header,
+	// @see: ffmpeg, AVCodecContext::extradata
 	int 		aac_extra_size;
 	char*		aac_extra_data;
 public:
@@ -170,7 +208,8 @@ public:
 	virtual ~SrsCodec();
 // the following function used for hls to build the codec info.
 public:
-	virtual int parse_av_codec(bool is_video, int8_t* data, int size);
+	virtual int parse_audio_codec(int8_t* data, int size);
+	virtual int parse_video_codec(int8_t* data, int size);
 // the following function used to finger out the flv/rtmp packet detail.
 public:
 	/**

trunk/src/core/srs_core_error.hpp

@@ -107,4 +107,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 // when open ssl sha256 digest key invalid size.
 #define ERROR_OpenSslSha256DigestSize	512
 
+#define ERROR_HLS_METADATA				600
+#define ERROR_HLS_DECODE_ERROR			601
+
 #endif

trunk/src/core/srs_core_hls.cpp

@@ -53,11 +53,13 @@ int SrsHLS::on_meta_data(SrsOnMetaDataPacket* metadata)
 	int ret = ERROR_SUCCESS;
 
 	if (!metadata || !metadata->metadata) {
+		srs_trace("no metadata persent, hls ignored it.");
 		return ret;
 	}
 	
 	SrsAmf0Object* obj = metadata->metadata;
 	if (obj->size() <= 0) {
+		srs_trace("no metadata persent, hls ignored it.");
 		return ret;
 	}
 	
@@ -90,21 +92,38 @@ int SrsHLS::on_meta_data(SrsOnMetaDataPacket* metadata)
 		codec->audio_data_rate = (int)(1000 * srs_amf0_convert<SrsAmf0Number>(prop)->value);
 	}
 	if ((prop = obj->get_property("audiosamplerate")) != NULL && prop->is_number()) {
-		codec->sample_rate = (int)srs_amf0_convert<SrsAmf0Number>(prop)->value;
+		int sound_rate = (int)srs_amf0_convert<SrsAmf0Number>(prop)->value;
+		if (sound_rate == 5512) {
+			codec->sound_rate = SrsCodecAudioSampleRate5512;
+		} else if (sound_rate == 11025) {
+			codec->sound_rate = SrsCodecAudioSampleRate11025;
+		} else if (sound_rate == 22050) {
+			codec->sound_rate = SrsCodecAudioSampleRate22050;
+		} else if (sound_rate == 44100) {
+			codec->sound_rate = SrsCodecAudioSampleRate44100;
+		} else {
+			ret = ERROR_HLS_METADATA;
+			srs_error("invalid sound_rate of metadata: %d, ret=%d", sound_rate, ret);
+			return ret;
+		}
 	}
 	if ((prop = obj->get_property("audiosamplesize")) != NULL && prop->is_number()) {
-		codec->sample_size = (int)srs_amf0_convert<SrsAmf0Number>(prop)->value;
+		int sound_size = (int)srs_amf0_convert<SrsAmf0Number>(prop)->value;
-		if (codec->sample_size == 16) {
+		if (sound_size == 16) {
-			codec->sample_size = 2;
+			codec->sound_size = SrsCodecAudioSampleSize16bit;
+		} else if (sound_size == 8) {
+			codec->sound_size = SrsCodecAudioSampleSize8bit;
 		} else {
-			codec->sample_size = 1;
+			ret = ERROR_HLS_METADATA;
+			srs_error("invalid sound_size of metadata: %d, ret=%d", sound_size, ret);
+			return ret;
 		}
 	}
 	if ((prop = obj->get_property("stereo")) != NULL && prop->is_number()) {
 		if (srs_amf0_convert<SrsAmf0Boolean>(prop)->value) {
-			codec->audio_channels = 2;
+			codec->sound_type = SrsCodecAudioSoundTypeStereo;
 		} else {
-			codec->audio_channels = 1;
+			codec->sound_type = SrsCodecAudioSoundTypeMono;
 		}
 	}
 	
@@ -115,7 +134,7 @@ int SrsHLS::on_audio(SrsCommonMessage* audio)
 {
 	int ret = ERROR_SUCCESS;
 	
-	if ((ret = codec->parse_av_codec(false, audio->payload, audio->size)) != ERROR_SUCCESS) {
+	if ((ret = codec->parse_audio_codec(audio->payload, audio->size)) != ERROR_SUCCESS) {
 		return ret;
 	}
 	
@@ -126,7 +145,7 @@ int SrsHLS::on_video(SrsCommonMessage* video)
 {
 	int ret = ERROR_SUCCESS;
 	
-	if ((ret = codec->parse_av_codec(true, video->payload, video->size)) != ERROR_SUCCESS) {
+	if ((ret = codec->parse_video_codec(video->payload, video->size)) != ERROR_SUCCESS) {
 		return ret;
 	}