From 4a5f479a0cbfcac1657b38e89740d1f461883d58 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Tue, 14 Feb 2023 14:28:41 +0800 Subject: [PATCH] GB: Support H.265 for GB28181 (#3408) Co-authored-by: chundonglinlin Co-authored-by: stone Co-authored-by: Winlin --- trunk/doc/CHANGELOG.md | 1 + trunk/src/app/srs_app_gb28181.cpp | 217 ++++++++++++++++++++++++++- trunk/src/app/srs_app_gb28181.hpp | 18 +++ trunk/src/app/srs_app_rtc_source.cpp | 7 + trunk/src/core/srs_core_version6.hpp | 2 +- trunk/src/kernel/srs_kernel_ts.cpp | 1 + 6 files changed, 242 insertions(+), 4 deletions(-) diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index dc62b16a9..17a2ca61b 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -8,6 +8,7 @@ The changelog for SRS. ## SRS 6.0 Changelog +* v6.0, 2023-02-14, Merge [#3408](https://github.com/ossrs/srs/pull/3408): GB: Support H.265 for GB28181. v6.0.25 (#3408) * v6.0, 2023-02-12, Merge [#3409](https://github.com/ossrs/srs/pull/3409): SRT: Reduce latency to 200ms of srt2rtc.conf. v6.0.24 (#3409) * v6.0, 2023-02-08, Merge [#3391](https://github.com/ossrs/srs/pull/3391): Config: Error when both HLS and HTTP-TS enabled. v6.0.23 (#3391) * v6.0, 2023-02-08, Merge [#3389](https://github.com/ossrs/srs/pull/3389): Kernel: Fix demux SPS error for NVENC and LARIX. v6.0.22 (#3389) diff --git a/trunk/src/app/srs_app_gb28181.cpp b/trunk/src/app/srs_app_gb28181.cpp index d54d6cf62..e65293259 100644 --- a/trunk/src/app/srs_app_gb28181.cpp +++ b/trunk/src/app/srs_app_gb28181.cpp @@ -169,6 +169,7 @@ void SrsLazyGbSession::on_ps_pack(SrsPackContext* ctx, SrsPsPacket* ps, const st // Group all videos to one video. if (msg->sid == SrsTsPESStreamIdVideoCommon) { + video->ps_helper_ = msg->ps_helper_; video->dts = msg->dts; video->pts = msg->pts; video->sid = msg->sid; @@ -1554,6 +1555,12 @@ SrsGbMuxer::SrsGbMuxer(SrsLazyGbSession* session) h264_pps_changed_ = false; h264_sps_pps_sent_ = false; +#ifdef SRS_H265 + hevc_ = new SrsRawHEVCStream(); + vps_sps_pps_sent_ = false; + vps_sps_pps_change_ = false; +#endif + aac_ = new SrsRawAacStream(); queue_ = new SrsMpegpsQueue(); @@ -1565,6 +1572,9 @@ SrsGbMuxer::~SrsGbMuxer() close(); srs_freep(avc_); +#ifdef SRS_H265 + srs_freep(hevc_); +#endif srs_freep(aac_); srs_freep(queue_); srs_freep(pprint_); @@ -1606,6 +1616,30 @@ srs_error_t SrsGbMuxer::on_ts_video(SrsTsMessage* msg, SrsBuffer* avs) return srs_error_wrap(err, "connect"); } + SrsPsDecodeHelper* h = (SrsPsDecodeHelper*)msg->ps_helper_; + srs_assert(h && h->ctx_ && h->ps_); + + if (h->ctx_->video_stream_type_ == SrsTsStreamVideoH264) { + if ((err = mux_h264(msg, avs)) != srs_success){ + return srs_error_wrap(err, "mux h264"); + } +#ifdef SRS_H265 + } else if (h->ctx_->video_stream_type_ == SrsTsStreamVideoHEVC) { + if ((err = mux_h265(msg, avs)) != srs_success){ + return srs_error_wrap(err, "mux hevc"); + } +#endif + } else { + return srs_error_new(ERROR_STREAM_CASTER_TS_CODEC, "ts: unsupported stream codec=%d", h->ctx_->video_stream_type_); + } + + return err; +} + +srs_error_t SrsGbMuxer::mux_h264(SrsTsMessage *msg, SrsBuffer *avs) +{ + srs_error_t err = srs_success; + // ts tbn to flv tbn. uint32_t dts = (uint32_t)(msg->dts / 90); uint32_t pts = (uint32_t)(msg->dts / 90); @@ -1615,7 +1649,7 @@ srs_error_t SrsGbMuxer::on_ts_video(SrsTsMessage* msg, SrsBuffer* avs) char* frame = NULL; int frame_size = 0; if ((err = avc_->annexb_demux(avs, &frame, &frame_size)) != srs_success) { - return srs_error_wrap(err, "demux annexb"); + return srs_error_wrap(err, "demux avc annexb"); } // 5bits, 7.3.1 NAL unit syntax, @@ -1764,6 +1798,183 @@ srs_error_t SrsGbMuxer::write_h264_ipb_frame(char* frame, int frame_size, uint32 return rtmp_write_packet(SrsFrameTypeVideo, timestamp, flv, nb_flv); } +#ifdef SRS_H265 +srs_error_t SrsGbMuxer::mux_h265(SrsTsMessage *msg, SrsBuffer *avs) +{ + srs_error_t err = srs_success; + + // ts tbn to flv tbn. + uint32_t dts = (uint32_t)(msg->dts / 90); + uint32_t pts = (uint32_t)(msg->dts / 90); + + // send each frame. + while (!avs->empty()) { + char* frame = NULL; + int frame_size = 0; + if ((err = hevc_->annexb_demux(avs, &frame, &frame_size)) != srs_success) { + return srs_error_wrap(err, "demux hevc annexb"); + } + + // 6bits, 7.4.2.2 NAL unit header semantics + // ITU-T-H.265-2021.pdf, page 85. + // 32: VPS, 33: SPS, 34: PPS ... + SrsHevcNaluType nt = SrsHevcNaluTypeParse(frame[0]); + if (nt == SrsHevcNaluType_SEI || nt == SrsHevcNaluType_SEI_SUFFIX || nt == SrsHevcNaluType_ACCESS_UNIT_DELIMITER) { + continue; + } + + // for vps + if (hevc_->is_vps(frame, frame_size)) { + std::string vps; + if ((err = hevc_->vps_demux(frame, frame_size, vps)) != srs_success) { + return srs_error_wrap(err, "demux vps"); + } + + if (h265_vps_ == vps) { + continue; + } + + vps_sps_pps_change_ = true; + h265_vps_ = vps; + + if ((err = write_h265_vps_sps_pps(dts, pts)) != srs_success) { + return srs_error_wrap(err, "write vps"); + } + continue; + } + + // for sps + if (hevc_->is_sps(frame, frame_size)) { + std::string sps; + if ((err = hevc_->sps_demux(frame, frame_size, sps)) != srs_success) { + return srs_error_wrap(err, "demux sps"); + } + + if (h265_sps_ == sps) { + continue; + } + vps_sps_pps_change_ = true; + h265_sps_ = sps; + + if ((err = write_h265_vps_sps_pps(dts, pts)) != srs_success) { + return srs_error_wrap(err, "write sps"); + } + continue; + } + + // for pps + if (hevc_->is_pps(frame, frame_size)) { + std::string pps; + if ((err = hevc_->pps_demux(frame, frame_size, pps)) != srs_success) { + return srs_error_wrap(err, "demux pps"); + } + + if (h265_pps_ == pps) { + continue; + } + vps_sps_pps_change_ = true; + h265_pps_ = pps; + + if ((err = write_h265_vps_sps_pps(dts, pts)) != srs_success) { + return srs_error_wrap(err, "write pps"); + } + continue; + } + + // ibp frame. + // TODO: FIXME: we should group all frames to a rtmp/flv message from one ts message. + srs_info("Muxer: demux avc ibp frame size=%d, dts=%d", frame_size, dts); + if ((err = write_h265_ipb_frame(frame, frame_size, dts, pts)) != srs_success) { + return srs_error_wrap(err, "write frame"); + } + } + + return err; +} + +srs_error_t SrsGbMuxer::write_h265_vps_sps_pps(uint32_t dts, uint32_t pts) +{ + srs_error_t err = srs_success; + + if (!vps_sps_pps_change_){ + return err; + } + + if (h265_vps_.empty() || h265_sps_.empty() || h265_pps_.empty()) { + return err; + } + + std::string sh; + if ((err = hevc_->mux_sequence_header(h265_vps_, h265_sps_, h265_pps_, sh)) != srs_success) { + return srs_error_wrap(err, "hevc mux sequence header"); + } + + // h265 packet to flv packet. + int8_t frame_type = SrsVideoAvcFrameTypeKeyFrame; + int8_t hevc_packet_type = SrsVideoAvcFrameTraitSequenceHeader; + + char* flv = NULL; + int nb_flv = 0; + + if ((err = hevc_->mux_avc2flv(sh, frame_type, hevc_packet_type, dts, pts, &flv, &nb_flv)) != srs_success) { + return srs_error_wrap(err, "hevc to flv"); + } + + // the timestamp in rtmp message header is dts. + uint32_t timestamp = dts; + if ((err = rtmp_write_packet(SrsFrameTypeVideo, timestamp, flv, nb_flv)) != srs_success) { + return srs_error_wrap(err, "hevc write packet"); + } + + // reset vps/sps/pps. + vps_sps_pps_change_ = false; + vps_sps_pps_sent_ = true; + + return err; +} + + +srs_error_t SrsGbMuxer::write_h265_ipb_frame(char* frame, int frame_size, uint32_t dts, uint32_t pts) +{ + srs_error_t err = srs_success; + + // when sps or pps not sent, ignore the packet. + if (!vps_sps_pps_sent_) { + return srs_error_new(ERROR_H264_DROP_BEFORE_SPS_PPS, "drop for no vps/sps/pps"); + } + + SrsHevcNaluType nt = SrsHevcNaluTypeParse(frame[0]); + + // F.3.29 intra random access point (IRAP) picture + // ITU-T-H.265-2021.pdf, page 28. + SrsVideoAvcFrameType frame_type = SrsVideoAvcFrameTypeInterFrame; + if (nt >= SrsHevcNaluType_CODED_SLICE_BLA || nt <= SrsHevcNaluType_RESERVED_23) { + frame_type = SrsVideoAvcFrameTypeKeyFrame; + } + + string ipb; + if ((err = hevc_->mux_ipb_frame(frame, frame_size, ipb)) != srs_success){ + return srs_error_wrap(err, "hevc mux ipb frame"); + } + + int8_t hevc_packet_type = SrsVideoAvcFrameTraitNALU; + char* flv = NULL; + int nb_flv = 0; + + if ((err = hevc_->mux_avc2flv(ipb, frame_type, hevc_packet_type, dts, pts, &flv, &nb_flv)) != srs_success) { + return srs_error_wrap(err, "hevc to flv"); + } + + // the timestamp in rtmp message header is dts. + uint32_t timestamp = dts; + if (( err = rtmp_write_packet(SrsFrameTypeVideo, timestamp, flv, nb_flv)) != srs_success){ + return srs_error_wrap(err, "hevc write packet"); + } + + return err; +} +#endif + srs_error_t SrsGbMuxer::on_ts_audio(SrsTsMessage* msg, SrsBuffer* avs) { srs_error_t err = srs_success; @@ -2379,12 +2590,12 @@ srs_error_t SrsRecoverablePsContext::decode(SrsBuffer* stream, ISrsPsMessageHand if ((err = ctx_.decode(stream, handler)) != srs_success) { return enter_recover_mode(stream, handler, stream->pos(), srs_error_wrap(err, "decode pack")); } - +#ifndef SRS_H265 // Check stream type, error if HEVC, because not supported yet. if (ctx_.video_stream_type_ == SrsTsStreamVideoHEVC) { return srs_error_new(ERROR_GB_PS_HEADER, "HEVC is not supported"); } - +#endif return err; } diff --git a/trunk/src/app/srs_app_gb28181.hpp b/trunk/src/app/srs_app_gb28181.hpp index 1ff8adf39..8876ed7bd 100644 --- a/trunk/src/app/srs_app_gb28181.hpp +++ b/trunk/src/app/srs_app_gb28181.hpp @@ -36,6 +36,9 @@ class SrsGbMuxer; class SrsSimpleRtmpClient; struct SrsRawAacStreamCodec; class SrsRawH264Stream; +#ifdef SRS_H265 +class SrsRawHEVCStream; +#endif class SrsSharedPtrMessage; class SrsPithyPrint; class SrsRawAacStream; @@ -409,6 +412,15 @@ private: std::string h264_pps_; bool h264_pps_changed_; bool h264_sps_pps_sent_; + +#ifdef SRS_H265 + SrsRawHEVCStream* hevc_; + bool vps_sps_pps_change_; + std::string h265_vps_; + std::string h265_sps_; + std::string h265_pps_; + bool vps_sps_pps_sent_; +#endif private: SrsRawAacStream* aac_; std::string aac_specific_config_; @@ -423,8 +435,14 @@ public: srs_error_t on_ts_message(SrsTsMessage* msg); private: virtual srs_error_t on_ts_video(SrsTsMessage* msg, SrsBuffer* avs); + virtual srs_error_t mux_h264(SrsTsMessage* msg, SrsBuffer* avs); virtual srs_error_t write_h264_sps_pps(uint32_t dts, uint32_t pts); virtual srs_error_t write_h264_ipb_frame(char* frame, int frame_size, uint32_t dts, uint32_t pts); +#ifdef SRS_H265 + virtual srs_error_t mux_h265(SrsTsMessage* msg, SrsBuffer* avs); + virtual srs_error_t write_h265_vps_sps_pps(uint32_t dts, uint32_t pts); + virtual srs_error_t write_h265_ipb_frame(char* frame, int frame_size, uint32_t dts, uint32_t pts); +#endif virtual srs_error_t on_ts_audio(SrsTsMessage* msg, SrsBuffer* avs); virtual srs_error_t write_audio_raw_frame(char* frame, int frame_size, SrsRawAacStreamCodec* codec, uint32_t dts); virtual srs_error_t rtmp_write_packet(char type, uint32_t timestamp, char* data, int size); diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index f18b9518e..3e023bf9e 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -974,6 +974,13 @@ srs_error_t SrsRtcFromRtmpBridge::on_video(SrsSharedPtrMessage* msg) return err; } + // WebRTC NOT support HEVC. +#ifdef SRS_H265 + if (format->vcodec->id == SrsVideoCodecIdHEVC) { + return err; + } +#endif + // cache the sequence header if h264 bool is_sequence_header = SrsFlvVideo::sh(msg->payload, msg->size); if (is_sequence_header && (err = meta->update_vsh(msg)) != srs_success) { diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp index ff01f3916..c6b35a0bd 100644 --- a/trunk/src/core/srs_core_version6.hpp +++ b/trunk/src/core/srs_core_version6.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 6 #define VERSION_MINOR 0 -#define VERSION_REVISION 24 +#define VERSION_REVISION 25 #endif diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 3a2ee5d8d..10f0c5c11 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -156,6 +156,7 @@ SrsTsMessage* SrsTsMessage::detach() { // @remark the packet cannot be used, but channel is ok. SrsTsMessage* cp = new SrsTsMessage(channel, NULL); + cp->ps_helper_ = ps_helper_; cp->start_pts = start_pts; cp->write_pcr = write_pcr; cp->is_discontinuity = is_discontinuity;