From 70d5618979e5c8dc41b7cd87c78db7ca2b8a10e8 Mon Sep 17 00:00:00 2001 From: Winlin Date: Wed, 23 Nov 2022 17:05:21 +0800 Subject: [PATCH] H265: Support HEVC over HTTP-TS. v6.0.4 (#3275) 1. Update TS video codec to HEVC during streaming. 2. Return error when HEVC is disabled. 3. Parse HEVC NALU type by SrsHevcNaluTypeParse. 4. Show message when codec change for TS. Co-authored-by: runner365 --- trunk/doc/CHANGELOG.md | 1 + trunk/src/app/srs_app_hls.cpp | 12 ++- trunk/src/app/srs_app_hls.hpp | 2 +- trunk/src/core/srs_core_version6.hpp | 2 +- trunk/src/kernel/srs_kernel_codec.cpp | 9 +- trunk/src/kernel/srs_kernel_codec.hpp | 1 + trunk/src/kernel/srs_kernel_error.hpp | 3 +- trunk/src/kernel/srs_kernel_ts.cpp | 140 ++++++++++++++++++++++---- trunk/src/kernel/srs_kernel_ts.hpp | 10 +- trunk/src/utest/srs_utest_kernel.cpp | 31 +++++- 10 files changed, 178 insertions(+), 33 deletions(-) diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index bafd48212..f46e27744 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -8,6 +8,7 @@ The changelog for SRS. ## SRS 6.0 Changelog +* v6.0, 2022-11-23, Merge [#3275](https://github.com/ossrs/srs/pull/3275): H265: Support HEVC over HTTP-TS. v6.0.4 * v6.0, 2022-11-23, Merge [#3274](https://github.com/ossrs/srs/pull/3274): H265: Support parse multiple NALUs in a frame. v6.0.3 * v6.0, 2022-11-22, Merge [#3272](https://github.com/ossrs/srs/pull/3272): H265: Support HEVC over RTMP or HTTP-FLV. v6.0.2 * v6.0, 2022-11-22, Merge [#3268](https://github.com/ossrs/srs/pull/3268): H265: Update mpegts.js to play HEVC over HTTP-TS/FLV. v6.0.1 diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 0183d1626..a1a544631 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -555,7 +555,7 @@ srs_error_t SrsHlsMuxer::flush_audio(SrsTsMessageCache* cache) return err; } -srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache) +srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame) { srs_error_t err = srs_success; @@ -573,6 +573,12 @@ srs_error_t SrsHlsMuxer::flush_video(SrsTsMessageCache* cache) // update the duration of segment. current->append(cache->video->dts / 90); + + // The video codec might change during streaming. Note that the frame might be NULL, when reap segment. + if (frame && frame->vcodec()) { + SrsTsContextWriter* tscw = current->tscw; + tscw->update_video_codec(frame->vcodec()->id); + } if ((err = current->tscw->write_video(cache->video)) != srs_success) { return srs_error_wrap(err, "hls: write video"); @@ -1025,7 +1031,7 @@ srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts) } // flush video when got one - if ((err = muxer->flush_video(tsmc)) != srs_success) { + if ((err = muxer->flush_video(tsmc, frame)) != srs_success) { return srs_error_wrap(err, "hls: flush video"); } @@ -1057,7 +1063,7 @@ srs_error_t SrsHlsController::reap_segment() } // segment open, flush video first. - if ((err = muxer->flush_video(tsmc)) != srs_success) { + if ((err = muxer->flush_video(tsmc, NULL)) != srs_success) { return srs_error_wrap(err, "hls: flush video"); } diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp index cd1eb7438..b5dd8a82a 100644 --- a/trunk/src/app/srs_app_hls.hpp +++ b/trunk/src/app/srs_app_hls.hpp @@ -193,7 +193,7 @@ public: // Whether current hls muxer is pure audio mode. virtual bool pure_audio(); virtual srs_error_t flush_audio(SrsTsMessageCache* cache); - virtual srs_error_t flush_video(SrsTsMessageCache* cache); + virtual srs_error_t flush_video(SrsTsMessageCache* cache, SrsVideoFrame* frame); // Close segment(ts). virtual srs_error_t segment_close(); private: diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp index e18455ca3..2ec200fc5 100644 --- a/trunk/src/core/srs_core_version6.hpp +++ b/trunk/src/core/srs_core_version6.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 6 #define VERSION_MINOR 0 -#define VERSION_REVISION 3 +#define VERSION_REVISION 4 #endif diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index df35dd54a..ef15211dc 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -620,11 +620,11 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size) // For HEVC(H.265), try to parse the IDR from NALUs. if (c && c->id == SrsVideoCodecIdHEVC) { #ifdef SRS_H265 - SrsHevcNaluType nalu_type = (SrsHevcNaluType)(uint8_t)((bytes[0] & 0x3f) >> 1); + SrsHevcNaluType nalu_type = SrsHevcNaluTypeParse(bytes[0]); has_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23); return err; #else - return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); #endif } @@ -854,7 +854,7 @@ srs_error_t SrsFormat::video_avc_demux(SrsBuffer* stream, int64_t timestamp) } return err; #else - return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); #endif } @@ -1373,7 +1373,7 @@ srs_error_t SrsFormat::video_nalu_demux(SrsBuffer* stream) // TODO: FIXME: Might need to guess format? return do_avc_demux_ibmf_format(stream); #else - return srs_error_new(ERROR_HLS_DECODE_ERROR, "H.265 is disabled"); + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); #endif } @@ -1500,6 +1500,7 @@ srs_error_t SrsFormat::do_avc_demux_ibmf_format(SrsBuffer* stream) // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 for (int i = 0; i < PictureLength;) { // unsigned int((NAL_unit_length+1)*8) NALUnitLength; + // TODO: FIXME: Should ignore error? See https://github.com/ossrs/srs-gb28181/commit/a13b9b54938a14796abb9011e7a8ee779439a452 if (!stream->require(vcodec->NAL_unit_length + 1)) { return srs_error_new(ERROR_HLS_DECODE_ERROR, "PictureLength:%d, i:%d, NaluLength:%d, left:%d", PictureLength, i, vcodec->NAL_unit_length, stream->left()); diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index aa7b32c51..682d4bca9 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -468,6 +468,7 @@ enum SrsHevcNaluType { SrsHevcNaluType_UNSPECIFIED_63, SrsHevcNaluType_INVALID, }; +#define SrsHevcNaluTypeParse(code) (SrsHevcNaluType)((code & 0x7E) >> 1) struct SrsHevcNalData { uint16_t nal_unit_length; diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index fee130c79..4a5553e29 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -266,7 +266,8 @@ XX(ERROR_INOTIFY_OPENFD , 3094, "InotifyOpenFd", "Failed to open inotify fd for config listener") \ XX(ERROR_INOTIFY_WATCH , 3095, "InotfyWatch", "Failed to watch inotify for config listener") \ XX(ERROR_HTTP_URL_UNESCAPE , 3096, "HttpUrlUnescape", "Failed to unescape URL for HTTP") \ - XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") + XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") \ + XX(ERROR_HEVC_DISABLED , 3098, "HevcDisabled", "HEVC is disabled") /**************************************************/ /* HTTP/StreamConverter protocol error. */ diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 2b3e43ab5..7dd659cc1 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -43,6 +43,9 @@ string srs_ts_stream2string(SrsTsStream stream) case SrsTsStreamAudioAC3: return "AC3"; case SrsTsStreamAudioDTS: return "AudioDTS"; case SrsTsStreamVideoH264: return "H.264"; +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: return "H.265"; +#endif case SrsTsStreamVideoMpeg4: return "MP4"; case SrsTsStreamAudioMpeg4: return "MP4A"; default: return "Other"; @@ -285,6 +288,14 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr vs = SrsTsStreamVideoH264; video_pid = TS_VIDEO_AVC_PID; break; + case SrsVideoCodecIdHEVC: +#ifdef SRS_H265 + vs = SrsTsStreamVideoHEVC; + video_pid = TS_VIDEO_AVC_PID; + break; +#else + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); +#endif case SrsVideoCodecIdDisabled: vs = SrsTsStreamReserved; break; @@ -296,7 +307,6 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr case SrsVideoCodecIdOn2VP6: case SrsVideoCodecIdOn2VP6WithAlphaChannel: case SrsVideoCodecIdScreenVideoVersion2: - case SrsVideoCodecIdHEVC: case SrsVideoCodecIdAV1: vs = SrsTsStreamReserved; break; @@ -335,10 +345,13 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no a/v stream, vcodec=%d, acodec=%d", vc, ac); } - // when any codec changed, write PAT/PMT table. + // When any codec changed, write PAT/PMT table. if (vcodec != vc || acodec != ac) { - vcodec = vc; - acodec = ac; + if (vcodec != SrsVideoCodecIdReserved || acodec != SrsAudioCodecIdReserved1) { + srs_trace("TS: Refresh PMT when vcodec=%d=>%d, acodec=%d=>%d", vcodec, vc, acodec, ac); + } + vcodec = vc; acodec = ac; + if ((err = encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) != srs_success) { return srs_error_wrap(err, "ts: encode PAT/PMT"); } @@ -355,8 +368,12 @@ srs_error_t SrsTsContext::encode(ISrsStreamWriter* writer, SrsTsMessage* msg, Sr srs_error_t SrsTsContext::encode_pat_pmt(ISrsStreamWriter* writer, int16_t vpid, SrsTsStream vs, int16_t apid, SrsTsStream as) { srs_error_t err = srs_success; - - if (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3) { + + bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC); +#endif + if (!codec_ok) { return srs_error_new(ERROR_HLS_NO_STREAM, "ts: no PID, vs=%d, as=%d", vs, as); } @@ -425,8 +442,12 @@ srs_error_t SrsTsContext::encode_pes(ISrsStreamWriter* writer, SrsTsMessage* msg if (msg->payload->length() == 0) { return err; } - - if (sid != SrsTsStreamVideoH264 && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC) { + + bool codec_ok = (sid == SrsTsStreamVideoH264 || sid == SrsTsStreamAudioAAC || sid == SrsTsStreamAudioMp3); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (sid == SrsTsStreamVideoHEVC); +#endif + if (!codec_ok) { srs_info("ts: ignore the unknown stream, sid=%d", sid); return err; } @@ -750,10 +771,14 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context, pmt->current_next_indicator = 1; pmt->section_number = 0; pmt->last_section_number = 0; - - // must got one valid codec. - srs_assert(vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); - + + // Here we must get the correct codec. + bool codec_ok = (vs == SrsTsStreamVideoH264 || as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC); +#endif + srs_assert(codec_ok); + // if mp3 or aac specified, use audio to carry pcr. if (as == SrsTsStreamAudioAAC || as == SrsTsStreamAudioMp3) { // use audio to carray pcr by default. @@ -762,8 +787,12 @@ SrsTsPacket* SrsTsPacket::create_pmt(SrsTsContext* context, pmt->infos.push_back(new SrsTsPayloadPMTESInfo(as, apid)); } - // if h.264 specified, use video to carry pcr. - if (vs == SrsTsStreamVideoH264) { + // If h.264/h.265 specified, use video to carry pcr. + codec_ok = (vs == SrsTsStreamVideoH264); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (vs == SrsTsStreamVideoHEVC); +#endif + if (codec_ok) { pmt->PCR_PID = vpid; pmt->infos.push_back(new SrsTsPayloadPMTESInfo(vs, vpid)); } @@ -2533,6 +2562,9 @@ srs_error_t SrsTsPayloadPMT::psi_decode(SrsBuffer* stream) // update the apply pid table switch (info->stream_type) { case SrsTsStreamVideoH264: +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: +#endif case SrsTsStreamVideoMpeg4: packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type); break; @@ -2616,6 +2648,9 @@ srs_error_t SrsTsPayloadPMT::psi_encode(SrsBuffer* stream) // update the apply pid table switch (info->stream_type) { case SrsTsStreamVideoH264: +#ifdef SRS_H265 + case SrsTsStreamVideoHEVC: +#endif case SrsTsStreamVideoMpeg4: packet->context->set(info->elementary_PID, SrsTsPidApplyVideo, info->stream_type); break; @@ -2685,6 +2720,11 @@ SrsVideoCodecId SrsTsContextWriter::video_codec() return vcodec; } +void SrsTsContextWriter::update_video_codec(SrsVideoCodecId v) +{ + vcodec = v; +} + SrsEncFileWriter::SrsEncFileWriter() { memset(iv,0,16); @@ -2832,8 +2872,17 @@ srs_error_t SrsTsMessageCache::cache_video(SrsVideoFrame* frame, int64_t dts) video->dts = dts; video->pts = video->dts + frame->cts * 90; video->sid = SrsTsPESStreamIdVideoCommon; - - // write video to cache. + + // Write H.265 video frame to cache. + if (frame && frame->vcodec()->id == SrsVideoCodecIdHEVC) { +#ifdef SRS_H265 + return do_cache_hevc(frame); +#else + return srs_error_new(ERROR_HEVC_DISABLED, "H.265 is disabled"); +#endif + } + + // Write H.264 video frame to cache. if ((err = do_cache_avc(frame)) != srs_success) { return srs_error_wrap(err, "ts: cache avc"); } @@ -2924,7 +2973,7 @@ srs_error_t SrsTsMessageCache::do_cache_aac(SrsAudioFrame* frame) return err; } -void srs_avc_insert_aud(SrsSimpleStream* payload, bool& aud_inserted) +void srs_avc_insert_aud(SrsSimpleStream* payload, bool aud_inserted) { // mux the samples in annexb format, // ISO_IEC_14496-10-AVC-2012.pdf, page 324. @@ -3064,6 +3113,52 @@ srs_error_t SrsTsMessageCache::do_cache_avc(SrsVideoFrame* frame) return err; } +#ifdef SRS_H265 +srs_error_t SrsTsMessageCache::do_cache_hevc(SrsVideoFrame* frame) +{ + srs_error_t err = srs_success; + + // Whether aud inserted. + bool aud_inserted = false; + + SrsVideoCodecConfig* codec = frame->vcodec(); + srs_assert(codec); + + bool is_sps_pps_appended = false; + + // all sample use cont nalu header, except the sps-pps before IDR frame. + for (int i = 0; i < frame->nb_samples; i++) { + SrsSample* sample = &frame->samples[i]; + int32_t size = sample->size; + + if (!sample->bytes || size <= 0) { + return srs_error_new(ERROR_HLS_AVC_SAMPLE_SIZE, "ts: invalid avc sample length=%d", size); + } + + // Insert aud before NALU for HEVC. + SrsHevcNaluType nalu_type = (SrsHevcNaluType)SrsHevcNaluTypeParse(sample->bytes[0]); + bool is_idr = (SrsHevcNaluType_CODED_SLICE_BLA <= nalu_type) && (nalu_type <= SrsHevcNaluType_RESERVED_23); + if (is_idr && !frame->has_sps_pps && !is_sps_pps_appended) { + for (size_t i = 0; i < codec->hevc_dec_conf_record_.nalu_vec.size(); i++) { + const SrsHevcHvccNalu& nalu = codec->hevc_dec_conf_record_.nalu_vec[i]; + if (nalu.num_nalus <= 0 || nalu.nal_data_vec.empty()) continue; + + srs_avc_insert_aud(video->payload, aud_inserted); + const SrsHevcNalData& data = nalu.nal_data_vec.at(0); + video->payload->append((char*)&data.nal_unit_data[0], (int)data.nal_unit_data.size()); + is_sps_pps_appended = true; + } + } + + // Insert the NALU to video in annexb. + srs_avc_insert_aud(video->payload, aud_inserted); + video->payload->append(sample->bytes, sample->size); + } + + return err; +} +#endif + SrsTsTransmuxer::SrsTsTransmuxer() { writer = NULL; @@ -3158,10 +3253,17 @@ srs_error_t SrsTsTransmuxer::write_video(int64_t timestamp, char* data, int size if (format->video->frame_type == SrsVideoAvcFrameTypeVideoInfoFrame) { return err; } - - if (format->vcodec->id != SrsVideoCodecIdAVC) { + + bool codec_ok = (format->vcodec->id != SrsVideoCodecIdAVC); +#ifdef SRS_H265 + codec_ok = codec_ok ? : (format->vcodec->id != SrsVideoCodecIdHEVC); +#endif + if (!codec_ok) { return err; } + + // The video codec might change during streaming. + tscw->update_video_codec(format->vcodec->id); // ignore sequence header if (format->video->frame_type == SrsVideoAvcFrameTypeKeyFrame && format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { diff --git a/trunk/src/kernel/srs_kernel_ts.hpp b/trunk/src/kernel/srs_kernel_ts.hpp index dd7bb83fd..0fd7acea2 100644 --- a/trunk/src/kernel/srs_kernel_ts.hpp +++ b/trunk/src/kernel/srs_kernel_ts.hpp @@ -131,6 +131,10 @@ enum SrsTsStream // ITU-T Rec. H.222.0 | ISO/IEC 13818-1 Reserved // 0x15-0x7F SrsTsStreamVideoH264 = 0x1b, +#ifdef SRS_H265 + // For HEVC(H.265). + SrsTsStreamVideoHEVC = 0x24, +#endif // User Private // 0x80-0xFF SrsTsStreamAudioAC3 = 0x81, @@ -1272,8 +1276,9 @@ public: // Write a video frame to ts, virtual srs_error_t write_video(SrsTsMessage* video); public: - // get the video codec of ts muxer. + // Get or update the video codec of ts muxer. virtual SrsVideoCodecId video_codec(); + virtual void update_video_codec(SrsVideoCodecId v); }; // Used for HLS Encryption @@ -1315,6 +1320,9 @@ private: virtual srs_error_t do_cache_mp3(SrsAudioFrame* frame); virtual srs_error_t do_cache_aac(SrsAudioFrame* frame); virtual srs_error_t do_cache_avc(SrsVideoFrame* frame); +#ifdef SRS_H265 + virtual srs_error_t do_cache_hevc(SrsVideoFrame* frame); +#endif }; // Transmux the RTMP stream to HTTP-TS stream. diff --git a/trunk/src/utest/srs_utest_kernel.cpp b/trunk/src/utest/srs_utest_kernel.cpp index a469a6106..6ba29930d 100644 --- a/trunk/src/utest/srs_utest_kernel.cpp +++ b/trunk/src/utest/srs_utest_kernel.cpp @@ -4921,9 +4921,6 @@ VOID TEST(KernelTSTest, CoverContextEncode) srs_error_t err = ctx.encode(&f, &m, SrsVideoCodecIdDisabled, SrsAudioCodecIdDisabled); HELPER_EXPECT_FAILED(err); - - err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus); - HELPER_EXPECT_FAILED(err); err = ctx.encode(&f, &m, SrsVideoCodecIdAV1, SrsAudioCodecIdOpus); HELPER_EXPECT_FAILED(err); @@ -4953,6 +4950,34 @@ VOID TEST(KernelTSTest, CoverContextEncode) } } +VOID TEST(KernelTSTest, CoverContextEncodeHEVC) +{ + srs_error_t err; + + SrsTsContext ctx; + MockTsHandler h; + +#ifndef SRS_H265 + if (true) { + MockSrsFileWriter f; + SrsTsMessage m; + + err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus); + HELPER_EXPECT_FAILED(err); + } +#endif + +#ifdef SRS_H265 + if (true) { + MockSrsFileWriter f; + SrsTsMessage m; + + err = ctx.encode(&f, &m, SrsVideoCodecIdHEVC, SrsAudioCodecIdOpus); + HELPER_EXPECT_SUCCESS(err); + } +#endif +} + VOID TEST(KernelTSTest, CoverContextDecode) { srs_error_t err;