fix the aac jump bug on iphone, correct the audio pts, use audio buffer and flush audio

README.md

@@ -34,7 +34,7 @@ For example, use ffmpeg to publish:
 step 5: play live stream <br/>
 <pre>
 rtmp url: rtmp://127.0.0.1:1935/live/livestream
-m3u8 url: http://127.0.0.1:1935/live/livestream.m3u8
+m3u8 url: http://127.0.0.1:80/live/livestream.m3u8
 </pre>
 
 ### Summary

trunk/conf/srs.conf

@@ -13,8 +13,8 @@ vhost __defaultVhost__ {
     gop_cache       on;
     hls             on;
     hls_path        ./objs/nginx/html;
-    hls_fragment    10;
-    hls_window      60;
+    hls_fragment    5;
+    hls_window      30;
 }
 # the vhost disabled.
 vhost removed.vhost.com {

trunk/src/core/srs_core_codec.cpp

@@ -159,7 +159,7 @@ int SrsCodec::audio_aac_demux(int8_t* data, int size, SrsCodecSample* sample)
 	
 	int8_t sound_type = sound_format & 0x01;
 	int8_t sound_size = (sound_format >> 1) & 0x01;
-	int8_t sound_rate = (sound_format >> 2) & 0x01;
+	int8_t sound_rate = (sound_format >> 2) & 0x03;
 	sound_format = (sound_format >> 4) & 0x0f;
 	
 	audio_codec_id = sound_format;
@@ -167,6 +167,25 @@ int SrsCodec::audio_aac_demux(int8_t* data, int size, SrsCodecSample* sample)
 	sample->sound_rate = (SrsCodecAudioSampleRate)sound_rate;
 	sample->sound_size = (SrsCodecAudioSampleSize)sound_size;
 	
+	// reset the sample rate by sequence header
+    static int aac_sample_rates[] = {
+    	96000, 88200, 64000, 48000,
+    	44100, 32000, 24000, 22050,
+    	16000, 12000, 11025,  8000,
+    	7350,     0,     0,    0
+    };
+    switch (aac_sample_rates[aac_sample_rate]) {
+        case 11025:
+        	sample->sound_rate = SrsCodecAudioSampleRate11025;
+        	break;
+        case 22050:
+        	sample->sound_rate = SrsCodecAudioSampleRate22050;
+        	break;
+        case 44100:
+        	sample->sound_rate = SrsCodecAudioSampleRate44100;
+        	break;
+    };
+    
 	// only support aac
 	if (audio_codec_id != SrsCodecAudioAAC) {
 		ret = ERROR_HLS_DECODE_ERROR;

trunk/src/core/srs_core_config.hpp

@@ -40,6 +40,10 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #define SRS_CONF_DEFAULT_HLS_PATH "./objs/nginx/html"
 #define SRS_CONF_DEFAULT_HLS_FRAGMENT 10
 #define SRS_CONF_DEFAULT_HLS_WINDOW 60
+// in ms, for HLS aac sync time.
+#define SRS_CONF_DEFAULT_AAC_SYNC 100
+// in ms, for HLS aac flush the audio
+#define SRS_CONF_DEFAULT_AAC_DELAY 300
 
 class SrsFileBuffer
 {

trunk/src/core/srs_core_hls.cpp

@@ -269,9 +269,6 @@ public:
 			}
 		}
 		
-		// write success, clear and free the buffer
-		buffer->free();
-		
 		return ret;
 	}
 private:
@@ -360,16 +357,79 @@ SrsM3u8Segment::~SrsM3u8Segment()
 	srs_freep(muxer);
 }
 
-SrsHLS::SrsHLS()
+SrsHlsAacJitter::SrsHlsAacJitter()
+{
+	base_pts = 0;
+	nb_samples = 0;
+
+	// TODO: config it, 0 means no adjust
+	sync_ms = SRS_CONF_DEFAULT_AAC_SYNC;
+}
+
+SrsHlsAacJitter::~SrsHlsAacJitter()
+{
+}
+
+int64_t SrsHlsAacJitter::on_buffer_start(int64_t flv_pts, int sample_rate)
+{
+	// 0 = 5.5 kHz = 5512 Hz
+	// 1 = 11 kHz = 11025 Hz
+	// 2 = 22 kHz = 22050 Hz
+	// 3 = 44 kHz = 44100 Hz
+	static int flv_sample_rates[] = {5512, 11025, 22050, 44100};
+	int flv_sample_rate = flv_sample_rates[sample_rate & 0x03];
+
+	// sync time set to 0, donot adjust the aac timestamp.
+	if (!sync_ms) {
+		return flv_pts;
+	}
+	
+	// @see: ngx_rtmp_hls_audio
+    /* TODO: We assume here AAC frame size is 1024
+     *       Need to handle AAC frames with frame size of 960 */
+	int64_t est_pts = base_pts + nb_samples * 90000LL * 1024LL / flv_sample_rate;
+	int64_t dpts = (int64_t) (est_pts - flv_pts);
+
+    if (dpts <= (int64_t) sync_ms * 90 && dpts >= (int64_t) sync_ms * -90) {
+        srs_info("HLS correct aac pts "
+        	"from %"PRId64" to %"PRId64", base=%"PRId64", nb_samples=%d, sample_rate=%d",
+        	flv_pts, est_pts, nb_samples, flv_sample_rate, base_pts);
+
+		nb_samples++;
+		
+		return est_pts;
+    }
+    
+    // resync
+	srs_trace("HLS aac resync, dpts=%"PRId64", pts=%"PRId64
+		", base=%"PRId64", nb_samples=%"PRId64", sample_rate=%d",
+		dpts, flv_pts, base_pts, nb_samples, flv_sample_rate);
+	
+    base_pts = flv_pts;
+    nb_samples = 1;
+    
+    return flv_pts;
+}
+
+void SrsHlsAacJitter::on_buffer_continue()
+{
+	nb_samples++;
+}
+
+SrsHls::SrsHls()
 {
 	hls_enabled = false;
 	codec = new SrsCodec();
 	sample = new SrsCodecSample();
 	current = NULL;
 	jitter = new SrsRtmpJitter();
+	aac_jitter = new SrsHlsAacJitter();
 	file_index = 0;
-	m3u8_dts = stream_dts = 0;
+	audio_buffer_start_pts = m3u8_dts = stream_dts = 0;
 	hls_fragment = hls_window = 0;
+
+	// TODO: config it.
+	audio_delay = SRS_CONF_DEFAULT_AAC_DELAY;
 	
 	audio_buffer = new SrsCodecBuffer();
 	video_buffer = new SrsCodecBuffer();
@@ -378,11 +438,12 @@ SrsHLS::SrsHLS()
 	video_frame = new SrsMpegtsFrame();
 }
 
-SrsHLS::~SrsHLS()
+SrsHls::~SrsHls()
 {
 	srs_freep(codec);
 	srs_freep(sample);
 	srs_freep(jitter);
+	srs_freep(aac_jitter);
 
 	std::vector<SrsM3u8Segment*>::iterator it;
 	for (it = segments.begin(); it != segments.end(); ++it) {
@@ -403,7 +464,7 @@ SrsHLS::~SrsHLS()
 	srs_freep(video_frame);
 }
 
-int SrsHLS::on_publish(std::string _vhost, std::string _app, std::string _stream)
+int SrsHls::on_publish(std::string _vhost, std::string _app, std::string _stream)
 {
 	int ret = ERROR_SUCCESS;
 
@@ -435,12 +496,12 @@ int SrsHLS::on_publish(std::string _vhost, std::string _app, std::string _stream
 	return ret;
 }
 
-void SrsHLS::on_unpublish()
+void SrsHls::on_unpublish()
 {
 	hls_enabled = false;
 }
 
-int SrsHLS::on_meta_data(SrsOnMetaDataPacket* metadata)
+int SrsHls::on_meta_data(SrsOnMetaDataPacket* metadata)
 {
 	int ret = ERROR_SUCCESS;
 
@@ -492,7 +553,7 @@ int SrsHLS::on_meta_data(SrsOnMetaDataPacket* metadata)
 	return ret;
 }
 
-int SrsHLS::on_audio(SrsSharedPtrMessage* audio)
+int SrsHls::on_audio(SrsSharedPtrMessage* audio)
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -523,18 +584,43 @@ int SrsHLS::on_audio(SrsSharedPtrMessage* audio)
 	
 	srs_assert(current);
 	
-	stream_dts = audio_frame->dts = audio_frame->pts = audio->header.timestamp * 90;
-	audio_frame->pid = TS_AUDIO_PID;
-	audio_frame->sid = TS_AUDIO_AAC;
+	// the pts calc from rtmp/flv header.
+	int64_t pts = audio->header.timestamp * 90;
 	
-	if ((ret = current->muxer->write_audio(audio_frame, audio_buffer, codec, sample)) != ERROR_SUCCESS) {
+	// flush if audio delay exceed
+	if (pts - audio_buffer_start_pts > audio_delay * 90) {
+		if ((ret = flush_audio()) != ERROR_SUCCESS) {
+			return ret;
+		}
+	}
+	
+	// start buffer, set the audio_frame
+	if (audio_buffer->size == 0) {
+		pts = aac_jitter->on_buffer_start(pts, sample->sound_rate);
+		
+		audio_frame->dts = audio_frame->pts = audio_buffer_start_pts = pts;
+		audio_frame->pid = TS_AUDIO_PID;
+		audio_frame->sid = TS_AUDIO_AAC;
+	} else {
+		aac_jitter->on_buffer_continue();
+	}
+	
+	// write audio to cache.
+	if ((ret = write_audio()) != ERROR_SUCCESS) {
 		return ret;
 	}
 	
+	// write cache to file.
+	if (audio_buffer->size > 1024 * 1024) {
+		if ((ret = flush_audio()) != ERROR_SUCCESS) {
+			return ret;
+		}
+	}
+	
 	return ret;
 }
 
-int SrsHLS::on_video(SrsSharedPtrMessage* video)
+int SrsHls::on_video(SrsSharedPtrMessage* video)
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -563,6 +649,11 @@ int SrsHLS::on_video(SrsSharedPtrMessage* video)
 		return ret;
 	}
 	
+	// write video to cache.
+	if ((ret = write_video()) != ERROR_SUCCESS) {
+		return ret;
+	}
+	
 	stream_dts = video_frame->dts = video->header.timestamp * 90;
 	video_frame->pts = video_frame->dts + sample->cts * 90;
 	video_frame->pid = TS_VIDEO_PID;
@@ -580,14 +671,17 @@ int SrsHLS::on_video(SrsSharedPtrMessage* video)
 	}
 	
 	srs_assert(current);
-	if ((ret = current->muxer->write_video(video_frame, video_buffer, codec, sample)) != ERROR_SUCCESS) {
+	if ((ret = current->muxer->write_video(video_frame, video_buffer)) != ERROR_SUCCESS) {
 		return ret;
 	}
+		
+	// write success, clear and free the buffer
+	video_buffer->free();
 	
 	return ret;
 }
 
-int SrsHLS::reopen()
+int SrsHls::reopen()
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -675,10 +769,19 @@ int SrsHLS::reopen()
 	}
 	srs_info("open HLS muxer success. vhost=%s, path=%s", vhost.c_str(), current->full_path.c_str());
 	
+	// segment open, flush the audio.
+	// @see: ngx_rtmp_hls_open_fragment
+    /* start fragment with audio to make iPhone happy */
+	if (current->muxer->fresh()) {
+		if ((ret = flush_audio()) != ERROR_SUCCESS) {
+			return ret;
+		}
+	}
+	
 	return ret;
 }
 
-int SrsHLS::refresh_m3u8()
+int SrsHls::refresh_m3u8()
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -708,7 +811,7 @@ int SrsHLS::refresh_m3u8()
 	return ret;
 }
 
-int SrsHLS::_refresh_m3u8(int& fd, std::string m3u8_file)
+int SrsHls::_refresh_m3u8(int& fd, std::string m3u8_file)
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -799,7 +902,7 @@ int SrsHLS::_refresh_m3u8(int& fd, std::string m3u8_file)
 	return ret;
 }
 
-int SrsHLS::create_dir()
+int SrsHls::create_dir()
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -822,41 +925,7 @@ int SrsHLS::create_dir()
 	return ret;
 }
 
-SrsTSMuxer::SrsTSMuxer()
-{
-	fd = -1;
-}
-
-SrsTSMuxer::~SrsTSMuxer()
-{
-	close();
-}
-
-int SrsTSMuxer::open(std::string _path)
-{
-	int ret = ERROR_SUCCESS;
-	
-	path = _path;
-	
-	close();
-	
-	int flags = O_CREAT|O_WRONLY|O_TRUNC;
-	mode_t mode = S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH;
-	if ((fd = ::open(path.c_str(), flags, mode)) < 0) {
-		ret = ERROR_HLS_OPEN_FAILED;
-		srs_error("open ts file %s failed. ret=%d", path.c_str(), ret);
-		return ret;
-	}
-
-	// write mpegts header
-	if ((ret = SrsMpegtsWriter::write_header(fd)) != ERROR_SUCCESS) {
-		return ret;
-	}
-	
-	return ret;
-}
-
-int SrsTSMuxer::write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer, SrsCodec* codec, SrsCodecSample* sample)
+int SrsHls::write_audio()
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -920,14 +989,10 @@ int SrsTSMuxer::write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_b
 		audio_buffer->append(buf->bytes, buf->size);
 	}
 	
-	if ((ret = SrsMpegtsWriter::write_frame(fd, audio_frame, audio_buffer)) != ERROR_SUCCESS) {
-		return ret;
-	}
-	
 	return ret;
 }
 
-int SrsTSMuxer::write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer, SrsCodec* codec, SrsCodecSample* sample)
+int SrsHls::write_video()
 {
 	int ret = ERROR_SUCCESS;
 	
@@ -991,6 +1056,81 @@ int SrsTSMuxer::write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_b
 		video_buffer->append(buf->bytes, buf->size);
 	}
 	
+	return ret;
+}
+
+int SrsHls::flush_audio()
+{
+	int ret = ERROR_SUCCESS;
+	
+	if (audio_buffer->size <= 0) {
+		return ret;
+	}
+	
+	if ((ret = current->muxer->write_audio(audio_frame, audio_buffer)) != ERROR_SUCCESS) {
+		return ret;
+	}
+	
+	// write success, clear and free the buffer
+	audio_buffer->free();
+
+	return ret;
+}
+
+SrsTSMuxer::SrsTSMuxer()
+{
+	fd = -1;
+	_fresh = false;
+}
+
+SrsTSMuxer::~SrsTSMuxer()
+{
+	close();
+}
+
+int SrsTSMuxer::open(std::string _path)
+{
+	int ret = ERROR_SUCCESS;
+	
+	path = _path;
+	
+	close();
+	
+	int flags = O_CREAT|O_WRONLY|O_TRUNC;
+	mode_t mode = S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH;
+	if ((fd = ::open(path.c_str(), flags, mode)) < 0) {
+		ret = ERROR_HLS_OPEN_FAILED;
+		srs_error("open ts file %s failed. ret=%d", path.c_str(), ret);
+		return ret;
+	}
+
+	// write mpegts header
+	if ((ret = SrsMpegtsWriter::write_header(fd)) != ERROR_SUCCESS) {
+		return ret;
+	}
+	
+	_fresh = true;
+	
+	return ret;
+}
+
+int SrsTSMuxer::write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer)
+{
+	int ret = ERROR_SUCCESS;
+	
+	if ((ret = SrsMpegtsWriter::write_frame(fd, audio_frame, audio_buffer)) != ERROR_SUCCESS) {
+		return ret;
+	}
+	
+	_fresh = false;
+	
+	return ret;
+}
+
+int SrsTSMuxer::write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer)
+{
+	int ret = ERROR_SUCCESS;
+	
 	if ((ret = SrsMpegtsWriter::write_frame(fd, video_frame, video_buffer)) != ERROR_SUCCESS) {
 		return ret;
 	}
@@ -1003,6 +1143,12 @@ void SrsTSMuxer::close()
 	if (fd > 0) {
 		::close(fd);
 		fd = -1;
+		_fresh = false;
 	}
 }
 
+bool SrsTSMuxer::fresh()
+{
+	return _fresh;
+}
+

trunk/src/core/srs_core_hls.hpp

@@ -64,10 +64,40 @@ struct SrsM3u8Segment
 	virtual ~SrsM3u8Segment();
 };
 
+/**
+* jitter correct for audio,
+* the sample rate 44100/32000 will lost precise,
+* when mp4/ts(tbn=90000) covert to flv/rtmp(1000),
+* so the Hls on ipad or iphone will corrupt,
+* @see nginx-rtmp: est_pts
+*/
+class SrsHlsAacJitter
+{
+private:
+	int64_t base_pts;
+	int64_t nb_samples;
+	int sync_ms;
+public:
+	SrsHlsAacJitter();
+	virtual ~SrsHlsAacJitter();
+	/**
+	* when buffer start, calc the "correct" pts for ts,
+	* @param flv_pts, the flv pts calc from flv header timestamp,
+	* @return the calc correct pts.
+	*/
+	virtual int64_t on_buffer_start(int64_t flv_pts, int sample_rate);
+	/**
+	* when buffer continue, muxer donot write to file,
+	* the audio buffer continue grow and donot need a pts,
+	* for the ts audio PES packet only has one pts at the first time.
+	*/
+	virtual void on_buffer_continue();
+};
+
 /**
 * write m3u8 hls.
 */
-class SrsHLS
+class SrsHls
 {
 private:
 	std::string vhost;
@@ -95,16 +125,20 @@ private:
 	SrsCodecBuffer* video_buffer;
 	// last known dts
 	int64_t stream_dts;
+	int64_t audio_buffer_start_pts;
 	// last segment dts in m3u8
 	int64_t m3u8_dts;
+	// in ms, audio delay to flush the audios.
+	int64_t audio_delay;
 private:
 	bool hls_enabled;
 	SrsCodec* codec;
 	SrsCodecSample* sample;
 	SrsRtmpJitter* jitter;
+	SrsHlsAacJitter* aac_jitter;
 public:
-	SrsHLS();
-	virtual ~SrsHLS();
+	SrsHls();
+	virtual ~SrsHls();
 public:
 	virtual int on_publish(std::string _vhost, std::string _app, std::string _stream);
 	virtual void on_unpublish();
@@ -116,6 +150,10 @@ private:
 	virtual int refresh_m3u8();
 	virtual int _refresh_m3u8(int& fd, std::string m3u8_file);
 	virtual int create_dir();
+private:
+	virtual int write_audio();
+	virtual int write_video();
+	virtual int flush_audio();
 };
 
 class SrsTSMuxer
@@ -123,14 +161,16 @@ class SrsTSMuxer
 private:
 	int fd;
 	std::string path;
+	bool _fresh;
 public:
 	SrsTSMuxer();
 	virtual ~SrsTSMuxer();
 public:
 	virtual int open(std::string _path);
-	virtual int write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer, SrsCodec* codec, SrsCodecSample* sample);
-	virtual int write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer, SrsCodec* codec, SrsCodecSample* sample);
+	virtual int write_audio(SrsMpegtsFrame* audio_frame, SrsCodecBuffer* audio_buffer);
+	virtual int write_video(SrsMpegtsFrame* video_frame, SrsCodecBuffer* video_buffer);
 	virtual void close();
+	virtual bool fresh();
 };
 
 #endif

trunk/src/core/srs_core_source.cpp

@@ -257,7 +257,7 @@ SrsSource* SrsSource::find(std::string stream_url)
 SrsSource::SrsSource(std::string _stream_url)
 {
 	stream_url = _stream_url;
-	hls = new SrsHLS();
+	hls = new SrsHls();
 	
 	cache_metadata = cache_sh_video = cache_sh_audio = NULL;
 	

trunk/src/core/srs_core_source.hpp

@@ -38,7 +38,7 @@ class SrsSource;
 class SrsCommonMessage;
 class SrsOnMetaDataPacket;
 class SrsSharedPtrMessage;
-class SrsHLS;
+class SrsHls;
 
 /**
 * time jitter detect and correct,
@@ -125,7 +125,7 @@ public:
 	*/
 	static SrsSource* find(std::string stream_url);
 private:
-	SrsHLS* hls;
+	SrsHls* hls;
 	std::string stream_url;
 	std::vector<SrsConsumer*> consumers;
 // gop cache for client fast startup.