From edba2c25f13c0fa915bd8e8093a4005df6077858 Mon Sep 17 00:00:00 2001 From: mapengfei53 Date: Sun, 8 Jan 2023 15:05:43 +0800 Subject: [PATCH] HEVC: Support DVR HEVC stream to MP4. v6.0.14 (#3360) * DVR: Support mp4 blackbox test based on hooks. * HEVC: Support DASH HEVC stream * Refine blackbox test. v6.0.14 Co-authored-by: pengfei.ma Co-authored-by: winlin --- trunk/3rdparty/srs-bench/blackbox/dvr_test.go | 109 ++++++++ .../3rdparty/srs-bench/blackbox/hevc_test.go | 232 ++++++++++++++++++ trunk/doc/CHANGELOG.md | 1 + trunk/src/core/srs_core_version6.hpp | 2 +- trunk/src/kernel/srs_kernel_error.hpp | 4 +- trunk/src/kernel/srs_kernel_mp4.cpp | 193 ++++++++++++--- trunk/src/kernel/srs_kernel_mp4.hpp | 31 ++- trunk/src/utest/srs_utest_kernel.cpp | 5 + trunk/src/utest/srs_utest_mp4.cpp | 53 +++- 9 files changed, 587 insertions(+), 43 deletions(-) diff --git a/trunk/3rdparty/srs-bench/blackbox/dvr_test.go b/trunk/3rdparty/srs-bench/blackbox/dvr_test.go index 48214ceda..fbe55e32f 100644 --- a/trunk/3rdparty/srs-bench/blackbox/dvr_test.go +++ b/trunk/3rdparty/srs-bench/blackbox/dvr_test.go @@ -141,3 +141,112 @@ func TestFast_RtmpPublish_DvrFlv_Basic(t *testing.T) { } } } + +func TestFast_RtmpPublish_DvrMp4_Basic(t *testing.T) { + // This case is run in parallel. + t.Parallel() + + // Setup the max timeout for this case. + ctx, cancel := context.WithTimeout(logger.WithContext(context.Background()), time.Duration(*srsTimeout)*time.Millisecond) + defer cancel() + + // Check a set of errors. + var r0, r1, r2, r3, r4, r5, r6 error + defer func(ctx context.Context) { + if err := filterTestError(ctx.Err(), r0, r1, r2, r3, r4, r5, r6); err != nil { + t.Errorf("Fail for err %+v", err) + } else { + logger.Tf(ctx, "test done with err %+v", err) + } + }(ctx) + + var wg sync.WaitGroup + defer wg.Wait() + + // Start hooks service. + hooks := NewHooksService() + wg.Add(1) + go func() { + defer wg.Done() + r6 = hooks.Run(ctx, cancel) + }() + + // Start SRS server and wait for it to be ready. + svr := NewSRSServer(func(v *srsServer) { + v.envs = []string{ + "SRS_VHOST_DVR_ENABLED=on", + "SRS_VHOST_DVR_DVR_PLAN=session", + "SRS_VHOST_DVR_DVR_PATH=./objs/nginx/html/[app]/[stream].[timestamp].mp4", + fmt.Sprintf("SRS_VHOST_DVR_DVR_DURATION=%v", *srsFFprobeDuration), + "SRS_VHOST_HTTP_HOOKS_ENABLED=on", + fmt.Sprintf("SRS_VHOST_HTTP_HOOKS_ON_DVR=http://localhost:%v/api/v1/dvrs", hooks.HooksAPI()), + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-hooks.ReadyCtx().Done() + r0 = svr.Run(ctx, cancel) + }() + + // Start FFmpeg to publish stream. + duration := time.Duration(*srsFFprobeDuration) * time.Millisecond + streamID := fmt.Sprintf("stream-%v-%v", os.Getpid(), rand.Int()) + streamURL := fmt.Sprintf("rtmp://localhost:%v/live/%v", svr.RTMPPort(), streamID) + ffmpeg := NewFFmpeg(func(v *ffmpegClient) { + // When process quit, still keep case to run. + v.cancelCaseWhenQuit, v.ffmpegDuration = false, duration + v.args = []string{ + "-stream_loop", "-1", "-re", "-i", *srsPublishAvatar, "-c", "copy", "-f", "flv", streamURL, + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r1 = ffmpeg.Run(ctx, cancel) + }() + + // Start FFprobe to detect and verify stream. + ffprobe := NewFFprobe(func(v *ffprobeClient) { + v.dvrByFFmpeg, v.streamURL = false, streamURL + v.duration, v.timeout = duration, time.Duration(*srsFFprobeTimeout)*time.Millisecond + + wg.Add(1) + go func() { + defer wg.Done() + for evt := range hooks.HooksEvents() { + if onDvrEvt, ok := evt.(*HooksEventOnDvr); ok { + fp := path.Join(svr.WorkDir(), onDvrEvt.File) + logger.Tf(ctx, "FFprobe: Set the dvrFile=%v from callback", fp) + v.dvrFile = fp + } + } + }() + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r2 = ffprobe.Run(ctx, cancel) + }() + + // Fast quit for probe done. + select { + case <-ctx.Done(): + case <-ffprobe.ProbeDoneCtx().Done(): + defer cancel() + + str, m := ffprobe.Result() + if len(m.Streams) != 2 { + r3 = errors.Errorf("invalid streams=%v, %v, %v", len(m.Streams), m.String(), str) + } + + if ts := 90; m.Format.ProbeScore < ts { + r4 = errors.Errorf("low score=%v < %v, %v, %v", m.Format.ProbeScore, ts, m.String(), str) + } + if dv := m.Duration(); dv < duration/2 { + r5 = errors.Errorf("short duration=%v < %v, %v, %v", dv, duration/2, m.String(), str) + } + } +} diff --git a/trunk/3rdparty/srs-bench/blackbox/hevc_test.go b/trunk/3rdparty/srs-bench/blackbox/hevc_test.go index e313d19ef..01c5bac5b 100644 --- a/trunk/3rdparty/srs-bench/blackbox/hevc_test.go +++ b/trunk/3rdparty/srs-bench/blackbox/hevc_test.go @@ -412,3 +412,235 @@ func TestSlow_RtmpPublish_HlsPlay_HEVC_Basic(t *testing.T) { } } } + +func TestSlow_RtmpPublish_DvrFlv_HEVC_Basic(t *testing.T) { + // This case is run in parallel. + t.Parallel() + + // Setup the max timeout for this case. + ctx, cancel := context.WithTimeout(logger.WithContext(context.Background()), time.Duration(*srsTimeout)*time.Millisecond) + defer cancel() + + // Check a set of errors. + var r0, r1, r2, r3, r4, r5, r6 error + defer func(ctx context.Context) { + if err := filterTestError(ctx.Err(), r0, r1, r2, r3, r4, r5, r6); err != nil { + t.Errorf("Fail for err %+v", err) + } else { + logger.Tf(ctx, "test done with err %+v", err) + } + }(ctx) + + var wg sync.WaitGroup + defer wg.Wait() + + // Start hooks service. + hooks := NewHooksService() + wg.Add(1) + go func() { + defer wg.Done() + r6 = hooks.Run(ctx, cancel) + }() + + // Start SRS server and wait for it to be ready. + svr := NewSRSServer(func(v *srsServer) { + v.envs = []string{ + "SRS_VHOST_DVR_ENABLED=on", + "SRS_VHOST_DVR_DVR_PLAN=session", + "SRS_VHOST_DVR_DVR_PATH=./objs/nginx/html/[app]/[stream].[timestamp].flv", + fmt.Sprintf("SRS_VHOST_DVR_DVR_DURATION=%v", *srsFFprobeDuration), + "SRS_VHOST_HTTP_HOOKS_ENABLED=on", + fmt.Sprintf("SRS_VHOST_HTTP_HOOKS_ON_DVR=http://localhost:%v/api/v1/dvrs", hooks.HooksAPI()), + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-hooks.ReadyCtx().Done() + r0 = svr.Run(ctx, cancel) + }() + + // Start FFmpeg to publish stream. + duration := time.Duration(*srsFFprobeDuration) * time.Millisecond + streamID := fmt.Sprintf("stream-%v-%v", os.Getpid(), rand.Int()) + streamURL := fmt.Sprintf("rtmp://localhost:%v/live/%v", svr.RTMPPort(), streamID) + ffmpeg := NewFFmpeg(func(v *ffmpegClient) { + // When process quit, still keep case to run. + v.cancelCaseWhenQuit, v.ffmpegDuration = false, duration + v.args = []string{ + "-stream_loop", "-1", "-re", "-i", *srsPublishAvatar, "-acodec", "copy", "-vcodec", "libx265", + "-profile:v", "main", "-preset", "ultrafast", "-f", "flv", streamURL, + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r1 = ffmpeg.Run(ctx, cancel) + }() + + // Start FFprobe to detect and verify stream. + ffprobe := NewFFprobe(func(v *ffprobeClient) { + v.dvrByFFmpeg, v.streamURL = false, streamURL + v.duration, v.timeout = duration, time.Duration(*srsFFprobeTimeout)*time.Millisecond + + wg.Add(1) + go func() { + defer wg.Done() + for evt := range hooks.HooksEvents() { + if onDvrEvt, ok := evt.(*HooksEventOnDvr); ok { + fp := path.Join(svr.WorkDir(), onDvrEvt.File) + logger.Tf(ctx, "FFprobe: Set the dvrFile=%v from callback", fp) + v.dvrFile = fp + } + } + }() + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r2 = ffprobe.Run(ctx, cancel) + }() + + // Fast quit for probe done. + select { + case <-ctx.Done(): + case <-ffprobe.ProbeDoneCtx().Done(): + defer cancel() + + str, m := ffprobe.Result() + if len(m.Streams) != 2 { + r3 = errors.Errorf("invalid streams=%v, %v, %v", len(m.Streams), m.String(), str) + } + + if ts := 90; m.Format.ProbeScore < ts { + r4 = errors.Errorf("low score=%v < %v, %v, %v", m.Format.ProbeScore, ts, m.String(), str) + } + if dv := m.Duration(); dv < duration/2 { + r5 = errors.Errorf("short duration=%v < %v, %v, %v", dv, duration/2, m.String(), str) + } + + if v := m.Video(); v == nil { + r5 = errors.Errorf("no video %v, %v", m.String(), str) + } else if v.CodecName != "hevc" { + r6 = errors.Errorf("invalid video codec=%v, %v, %v", v.CodecName, m.String(), str) + } + } +} + +func TestSlow_RtmpPublish_DvrMp4_HEVC_Basic(t *testing.T) { + // This case is run in parallel. + t.Parallel() + + // Setup the max timeout for this case. + ctx, cancel := context.WithTimeout(logger.WithContext(context.Background()), time.Duration(*srsTimeout)*time.Millisecond) + defer cancel() + + // Check a set of errors. + var r0, r1, r2, r3, r4, r5, r6 error + defer func(ctx context.Context) { + if err := filterTestError(ctx.Err(), r0, r1, r2, r3, r4, r5, r6); err != nil { + t.Errorf("Fail for err %+v", err) + } else { + logger.Tf(ctx, "test done with err %+v", err) + } + }(ctx) + + var wg sync.WaitGroup + defer wg.Wait() + + // Start hooks service. + hooks := NewHooksService() + wg.Add(1) + go func() { + defer wg.Done() + r6 = hooks.Run(ctx, cancel) + }() + + // Start SRS server and wait for it to be ready. + svr := NewSRSServer(func(v *srsServer) { + v.envs = []string{ + "SRS_VHOST_DVR_ENABLED=on", + "SRS_VHOST_DVR_DVR_PLAN=session", + "SRS_VHOST_DVR_DVR_PATH=./objs/nginx/html/[app]/[stream].[timestamp].mp4", + fmt.Sprintf("SRS_VHOST_DVR_DVR_DURATION=%v", *srsFFprobeDuration), + "SRS_VHOST_HTTP_HOOKS_ENABLED=on", + fmt.Sprintf("SRS_VHOST_HTTP_HOOKS_ON_DVR=http://localhost:%v/api/v1/dvrs", hooks.HooksAPI()), + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-hooks.ReadyCtx().Done() + r0 = svr.Run(ctx, cancel) + }() + + // Start FFmpeg to publish stream. + duration := time.Duration(*srsFFprobeDuration) * time.Millisecond + streamID := fmt.Sprintf("stream-%v-%v", os.Getpid(), rand.Int()) + streamURL := fmt.Sprintf("rtmp://localhost:%v/live/%v", svr.RTMPPort(), streamID) + ffmpeg := NewFFmpeg(func(v *ffmpegClient) { + // When process quit, still keep case to run. + v.cancelCaseWhenQuit, v.ffmpegDuration = false, duration + v.args = []string{ + "-stream_loop", "-1", "-re", "-i", *srsPublishAvatar, "-acodec", "copy", "-vcodec", "libx265", + "-profile:v", "main", "-preset", "ultrafast", "-f", "flv", streamURL, + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r1 = ffmpeg.Run(ctx, cancel) + }() + + // Start FFprobe to detect and verify stream. + ffprobe := NewFFprobe(func(v *ffprobeClient) { + v.dvrByFFmpeg, v.streamURL = false, streamURL + v.duration, v.timeout = duration, time.Duration(*srsFFprobeTimeout)*time.Millisecond + + wg.Add(1) + go func() { + defer wg.Done() + for evt := range hooks.HooksEvents() { + if onDvrEvt, ok := evt.(*HooksEventOnDvr); ok { + fp := path.Join(svr.WorkDir(), onDvrEvt.File) + logger.Tf(ctx, "FFprobe: Set the dvrFile=%v from callback", fp) + v.dvrFile = fp + } + } + }() + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r2 = ffprobe.Run(ctx, cancel) + }() + + // Fast quit for probe done. + select { + case <-ctx.Done(): + case <-ffprobe.ProbeDoneCtx().Done(): + defer cancel() + + str, m := ffprobe.Result() + if len(m.Streams) != 2 { + r3 = errors.Errorf("invalid streams=%v, %v, %v", len(m.Streams), m.String(), str) + } + + if ts := 90; m.Format.ProbeScore < ts { + r4 = errors.Errorf("low score=%v < %v, %v, %v", m.Format.ProbeScore, ts, m.String(), str) + } + if dv := m.Duration(); dv < duration/2 { + r5 = errors.Errorf("short duration=%v < %v, %v, %v", dv, duration/2, m.String(), str) + } + + if v := m.Video(); v == nil { + r5 = errors.Errorf("no video %v, %v", m.String(), str) + } else if v.CodecName != "hevc" { + r6 = errors.Errorf("invalid video codec=%v, %v, %v", v.CodecName, m.String(), str) + } + } +} diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index 38fd9ad7d..d76f63cdc 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -8,6 +8,7 @@ The changelog for SRS. ## SRS 6.0 Changelog +* v6.0, 2023-01-08, Merge [#3360](https://github.com/ossrs/srs/pull/3360): H265: Support DVR HEVC stream to MP4. v6.0.14 * v6.0, 2023-01-06, Merge [#3363](https://github.com/ossrs/srs/issues/3363): HTTP: Add CORS Header for private network access. v6.0.13 * v6.0, 2023-01-04, Merge [#3362](https://github.com/ossrs/srs/issues/3362): SRT: Upgrade libsrt from 1.4.1 to 1.5.1. v6.0.12 * v6.0, 2023-01-02, For [#465](https://github.com/ossrs/srs/issues/465): HLS: Support HEVC over HLS. v6.0.11 diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp index b00210afe..0bc10cdbb 100644 --- a/trunk/src/core/srs_core_version6.hpp +++ b/trunk/src/core/srs_core_version6.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 6 #define VERSION_MINOR 0 -#define VERSION_REVISION 13 +#define VERSION_REVISION 14 #endif diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 48d1ca335..0e339702a 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -273,8 +273,8 @@ XX(ERROR_HTTP_URL_UNESCAPE , 3096, "HttpUrlUnescape", "Failed to unescape URL for HTTP") \ XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") \ XX(ERROR_HEVC_DISABLED , 3098, "HevcDisabled", "HEVC is disabled") \ - XX(ERROR_HEVC_DECODE_ERROR , 3099, "HevcDecode", "HEVC decode av stream failed") - + XX(ERROR_HEVC_DECODE_ERROR , 3099, "HevcDecode", "HEVC decode av stream failed") \ + XX(ERROR_MP4_HVCC_CHANGE , 3100, "Mp4HvcCChange", "MP4 does not support video HvcC change") /**************************************************/ /* HTTP/StreamConverter protocol error. */ #define SRS_ERRNO_MAP_HTTP(XX) \ diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index fc388a762..ccc09e62d 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -339,8 +339,10 @@ srs_error_t SrsMp4Box::discovery(SrsBuffer* buf, SrsMp4Box** ppbox) case SrsMp4BoxTypeSTCO: box = new SrsMp4ChunkOffsetBox(); break; case SrsMp4BoxTypeCO64: box = new SrsMp4ChunkLargeOffsetBox(); break; case SrsMp4BoxTypeSTSZ: box = new SrsMp4SampleSizeBox(); break; - case SrsMp4BoxTypeAVC1: box = new SrsMp4VisualSampleEntry(); break; + case SrsMp4BoxTypeAVC1: box = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); break; + case SrsMp4BoxTypeHEV1: box = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); break; case SrsMp4BoxTypeAVCC: box = new SrsMp4AvccBox(); break; + case SrsMp4BoxTypeHVCC: box = new SrsMp4HvcCBox(); break; case SrsMp4BoxTypeMP4A: box = new SrsMp4AudioSampleEntry(); break; case SrsMp4BoxTypeESDS: box = new SrsMp4EsdsBox(); break; case SrsMp4BoxTypeUDTA: box = new SrsMp4UserDataBox(); break; @@ -646,6 +648,14 @@ void SrsMp4FileTypeBox::set_compatible_brands(SrsMp4BoxBrand b0, SrsMp4BoxBrand compatible_brands[1] = b1; } +void SrsMp4FileTypeBox::set_compatible_brands(SrsMp4BoxBrand b0, SrsMp4BoxBrand b1, SrsMp4BoxBrand b2) +{ + compatible_brands.resize(3); + compatible_brands[0] = b0; + compatible_brands[1] = b1; + compatible_brands[2] = b2; +} + void SrsMp4FileTypeBox::set_compatible_brands(SrsMp4BoxBrand b0, SrsMp4BoxBrand b1, SrsMp4BoxBrand b2, SrsMp4BoxBrand b3) { compatible_brands.resize(4); @@ -3019,9 +3029,9 @@ stringstream& SrsMp4SampleEntry::dumps_detail(stringstream& ss, SrsMp4DumpContex return ss; } -SrsMp4VisualSampleEntry::SrsMp4VisualSampleEntry() : width(0), height(0) +SrsMp4VisualSampleEntry::SrsMp4VisualSampleEntry(SrsMp4BoxType boxType) : width(0), height(0) { - type = SrsMp4BoxTypeAVC1; + type = boxType; pre_defined0 = 0; reserved0 = 0; @@ -3051,6 +3061,18 @@ void SrsMp4VisualSampleEntry::set_avcC(SrsMp4AvccBox* v) boxes.push_back(v); } +SrsMp4HvcCBox* SrsMp4VisualSampleEntry::hvcC() +{ + SrsMp4Box* box = get(SrsMp4BoxTypeHVCC); + return dynamic_cast(box); +} + +void SrsMp4VisualSampleEntry::set_hvcC(SrsMp4HvcCBox* v) +{ + remove(SrsMp4BoxTypeHVCC); + boxes.push_back(v); +} + int SrsMp4VisualSampleEntry::nb_header() { return SrsMp4SampleEntry::nb_header()+2+2+12+2+2+4+4+4+2+32+2+2; @@ -3170,6 +3192,62 @@ stringstream& SrsMp4AvccBox::dumps_detail(stringstream& ss, SrsMp4DumpContext dc return ss; } +SrsMp4HvcCBox::SrsMp4HvcCBox() +{ + type = SrsMp4BoxTypeHVCC; +} + +SrsMp4HvcCBox::~SrsMp4HvcCBox() +{ +} + +int SrsMp4HvcCBox::nb_header() +{ + return SrsMp4Box::nb_header() + (int)hevc_config.size(); +} + +srs_error_t SrsMp4HvcCBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + if (!hevc_config.empty()) { + buf->write_bytes(&hevc_config[0], (int)hevc_config.size()); + } + + return err; +} + +srs_error_t SrsMp4HvcCBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + int nb_config = left_space(buf); + if (nb_config) { + hevc_config.resize(nb_config); + buf->read_bytes(&hevc_config[0], nb_config); + } + + return err; +} + +stringstream& SrsMp4HvcCBox::dumps_detail(stringstream& ss, SrsMp4DumpContext dc) +{ + SrsMp4Box::dumps_detail(ss, dc); + + ss << ", HEVC Config: " << (int)hevc_config.size() << "B" << endl; + srs_mp4_padding(ss, dc.indent()); + srs_mp4_print_bytes(ss, (const char*)&hevc_config[0], (int)hevc_config.size(), dc.indent()); + return ss; +} + SrsMp4AudioSampleEntry::SrsMp4AudioSampleEntry() : samplerate(0) { type = SrsMp4BoxTypeMP4A; @@ -5668,7 +5746,7 @@ srs_error_t SrsMp4Encoder::initialize(ISrsWriteSeeker* ws) ftyp->major_brand = SrsMp4BoxBrandISOM; ftyp->minor_version = 512; - ftyp->set_compatible_brands(SrsMp4BoxBrandISOM, SrsMp4BoxBrandISO2, SrsMp4BoxBrandAVC1, SrsMp4BoxBrandMP41); + ftyp->set_compatible_brands(SrsMp4BoxBrandISOM, SrsMp4BoxBrandISO2, SrsMp4BoxBrandMP41); int nb_data = ftyp->nb_bytes(); std::vector data(nb_data); @@ -5806,7 +5884,7 @@ srs_error_t SrsMp4Encoder::flush() mvhd->duration_in_tbn = srs_max(vduration, aduration); mvhd->next_track_ID = 1; // Starts from 1, increase when use it. - if (nb_videos || !pavcc.empty()) { + if (nb_videos || !pavcc.empty() || !phvcc.empty()) { SrsMp4TrackBox* trak = new SrsMp4TrackBox(); moov->add_trak(trak); @@ -5868,18 +5946,32 @@ srs_error_t SrsMp4Encoder::flush() SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); stbl->set_stsd(stsd); - - SrsMp4VisualSampleEntry* avc1 = new SrsMp4VisualSampleEntry(); - stsd->append(avc1); - - avc1->width = width; - avc1->height = height; - avc1->data_reference_index = 1; - - SrsMp4AvccBox* avcC = new SrsMp4AvccBox(); - avc1->set_avcC(avcC); - - avcC->avc_config = pavcc; + + if (vcodec == SrsVideoCodecIdAVC) { + SrsMp4VisualSampleEntry* avc1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); + stsd->append(avc1); + + avc1->width = width; + avc1->height = height; + avc1->data_reference_index = 1; + + SrsMp4AvccBox* avcC = new SrsMp4AvccBox(); + avc1->set_avcC(avcC); + + avcC->avc_config = pavcc; + } else { + SrsMp4VisualSampleEntry* hev1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); + stsd->append(hev1); + + hev1->width = width; + hev1->height = height; + hev1->data_reference_index = 1; + + SrsMp4HvcCBox* hvcC = new SrsMp4HvcCBox(); + hev1->set_hvcC(hvcC); + + hvcC->hevc_config = phvcc; + } } if (nb_audios || !pasc.empty()) { @@ -6036,13 +6128,24 @@ srs_error_t SrsMp4Encoder::flush() srs_error_t SrsMp4Encoder::copy_sequence_header(SrsFormat* format, bool vsh, uint8_t* sample, uint32_t nb_sample) { srs_error_t err = srs_success; - - if (vsh && !pavcc.empty()) { - if (nb_sample == (uint32_t)pavcc.size() && srs_bytes_equals(sample, &pavcc[0], (int)pavcc.size())) { - return err; + + if (vsh) { + // AVC + if (format->vcodec->id == SrsVideoCodecIdAVC && !pavcc.empty()) { + if (nb_sample == (uint32_t)pavcc.size() && srs_bytes_equals(sample, &pavcc[0], (int)pavcc.size())) { + return err; + } + + return srs_error_new(ERROR_MP4_AVCC_CHANGE, "doesn't support avcc change"); + } + // HEVC + if (format->vcodec->id == SrsVideoCodecIdHEVC && !phvcc.empty()) { + if (nb_sample == (uint32_t)phvcc.size() && srs_bytes_equals(sample, &phvcc[0], (int)phvcc.size())) { + return err; + } + + return srs_error_new(ERROR_MP4_HVCC_CHANGE, "doesn't support hvcC change"); } - - return srs_error_new(ERROR_MP4_AVCC_CHANGE, "doesn't support avcc change"); } if (!vsh && !pasc.empty()) { @@ -6054,7 +6157,11 @@ srs_error_t SrsMp4Encoder::copy_sequence_header(SrsFormat* format, bool vsh, uin } if (vsh) { - pavcc = std::vector(sample, sample + nb_sample); + if (format->vcodec->id == SrsVideoCodecIdHEVC) { + phvcc = std::vector(sample, sample + nb_sample); + } else { + pavcc = std::vector(sample, sample + nb_sample); + } if (format && format->vcodec) { width = format->vcodec->width; height = format->vcodec->height; @@ -6198,18 +6305,32 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); stbl->set_stsd(stsd); - - SrsMp4VisualSampleEntry* avc1 = new SrsMp4VisualSampleEntry(); - stsd->append(avc1); - - avc1->width = format->vcodec->width; - avc1->height = format->vcodec->height; - avc1->data_reference_index = 1; - - SrsMp4AvccBox* avcC = new SrsMp4AvccBox(); - avc1->set_avcC(avcC); - - avcC->avc_config = format->vcodec->avc_extra_data; + + if (format->vcodec->id == SrsVideoCodecIdAVC) { + SrsMp4VisualSampleEntry* avc1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); + stsd->append(avc1); + + avc1->width = format->vcodec->width; + avc1->height = format->vcodec->height; + avc1->data_reference_index = 1; + + SrsMp4AvccBox* avcC = new SrsMp4AvccBox(); + avc1->set_avcC(avcC); + + avcC->avc_config = format->vcodec->avc_extra_data; + } else { + SrsMp4VisualSampleEntry* hev1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); + stsd->append(hev1); + + hev1->width = format->vcodec->width; + hev1->height = format->vcodec->height; + hev1->data_reference_index = 1; + + SrsMp4HvcCBox* hvcC = new SrsMp4HvcCBox(); + hev1->set_hvcC(hvcC); + + hvcC->hevc_config = format->vcodec->avc_extra_data; + } SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); stbl->set_stts(stts); diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index 61ab82d58..4f203e437 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -32,7 +32,7 @@ class SrsMp4SampleDescriptionBox; class SrsMp4AvccBox; class SrsMp4DecoderSpecificInfo; class SrsMp4VisualSampleEntry; -class SrsMp4AvccBox; +class SrsMp4HvcCBox; class SrsMp4AudioSampleEntry; class SrsMp4EsdsBox; class SrsMp4ChunkOffsetBox; @@ -111,6 +111,8 @@ enum SrsMp4BoxType SrsMp4BoxTypeTFDT = 0x74666474, // 'tfdt' SrsMp4BoxTypeTRUN = 0x7472756e, // 'trun' SrsMp4BoxTypeSIDX = 0x73696478, // 'sidx' + SrsMp4BoxTypeHEV1 = 0x68657631, // 'hev1' + SrsMp4BoxTypeHVCC = 0x68766343, // 'hvcC' }; // 8.4.3.3 Semantics @@ -138,6 +140,7 @@ enum SrsMp4BoxBrand SrsMp4BoxBrandDASH = 0x64617368, // 'dash' SrsMp4BoxBrandMSDH = 0x6d736468, // 'msdh' SrsMp4BoxBrandMSIX = 0x6d736978, // 'msix' + SrsMp4BoxBrandHEV1 = 0x68657631, // 'hev1' }; // The context to dump. @@ -277,6 +280,7 @@ public: virtual ~SrsMp4FileTypeBox(); public: virtual void set_compatible_brands(SrsMp4BoxBrand b0, SrsMp4BoxBrand b1); + virtual void set_compatible_brands(SrsMp4BoxBrand b0, SrsMp4BoxBrand b1, SrsMp4BoxBrand b2); virtual void set_compatible_brands(SrsMp4BoxBrand b0, SrsMp4BoxBrand b1, SrsMp4BoxBrand b2, SrsMp4BoxBrand b3); protected: virtual int nb_header(); @@ -1261,12 +1265,16 @@ public: uint16_t depth; int16_t pre_defined2; public: - SrsMp4VisualSampleEntry(); + SrsMp4VisualSampleEntry(SrsMp4BoxType boxType); virtual ~SrsMp4VisualSampleEntry(); public: // For avc1, get the avcc box. virtual SrsMp4AvccBox* avcC(); virtual void set_avcC(SrsMp4AvccBox* v); +public: + // For hev1, get the hvcC box. + virtual SrsMp4HvcCBox* hvcC(); + virtual void set_hvcC(SrsMp4HvcCBox* v); protected: virtual int nb_header(); virtual srs_error_t encode_header(SrsBuffer* buf); @@ -1292,6 +1300,23 @@ public: virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); }; +// 8.4.1 HEVC Video Stream Definition (hvcC) +// ISO-14496-15-AVC-file-format-2017.pdf, page 73 +class SrsMp4HvcCBox : public SrsMp4Box +{ +public: + std::vector hevc_config; +public: + SrsMp4HvcCBox(); + virtual ~SrsMp4HvcCBox(); +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + // 8.5.2 Sample Description Box (mp4a) // ISO_IEC_14496-12-base-format-2012.pdf, page 45 class SrsMp4AudioSampleEntry : public SrsMp4SampleEntry @@ -2051,6 +2076,8 @@ public: private: // For H.264/AVC, the avcc contains the sps/pps. std::vector pavcc; + // For H.265/HEVC, the hvcC contains the vps/sps/pps. + std::vector phvcc; // The number of video samples. uint32_t nb_videos; // The duration of video stream. diff --git a/trunk/src/utest/srs_utest_kernel.cpp b/trunk/src/utest/srs_utest_kernel.cpp index 8171d31c2..e4674a92b 100644 --- a/trunk/src/utest/srs_utest_kernel.cpp +++ b/trunk/src/utest/srs_utest_kernel.cpp @@ -5441,6 +5441,7 @@ VOID TEST(KernelMP4Test, CoverMP4CodecSingleFrame) } enc.acodec = SrsAudioCodecIdAAC; + enc.vcodec = SrsVideoCodecIdAVC; HELPER_EXPECT_SUCCESS(enc.flush()); //mock_print_mp4(string(f.data(), f.filesize())); @@ -5557,6 +5558,7 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleVideos) } enc.acodec = SrsAudioCodecIdAAC; + enc.vcodec = SrsVideoCodecIdAVC; // Flush encoder. HELPER_EXPECT_SUCCESS(enc.flush()); @@ -5657,6 +5659,7 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleCTTs) } enc.acodec = SrsAudioCodecIdAAC; + enc.vcodec = SrsVideoCodecIdAVC; // Flush encoder. HELPER_EXPECT_SUCCESS(enc.flush()); @@ -5771,6 +5774,7 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleAVs) } enc.acodec = SrsAudioCodecIdAAC; + enc.vcodec = SrsVideoCodecIdAVC; // Flush encoder. HELPER_EXPECT_SUCCESS(enc.flush()); @@ -5889,6 +5893,7 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleAVsWithMp3) } enc.acodec = SrsAudioCodecIdMP3; + enc.vcodec = SrsVideoCodecIdAVC; // Flush encoder. HELPER_EXPECT_SUCCESS(enc.flush()); diff --git a/trunk/src/utest/srs_utest_mp4.cpp b/trunk/src/utest/srs_utest_mp4.cpp index 8c7a2f1f3..d642e8afc 100644 --- a/trunk/src/utest/srs_utest_mp4.cpp +++ b/trunk/src/utest/srs_utest_mp4.cpp @@ -1322,7 +1322,7 @@ VOID TEST(KernelMp4Test, SampleDescBox) SrsBuffer b(buf, sizeof(buf)); if (true) { - SrsMp4VisualSampleEntry box; + SrsMp4VisualSampleEntry box = SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); box.data_reference_index = 1; EXPECT_EQ((int)sizeof(buf), (int)box.nb_bytes()); HELPER_EXPECT_SUCCESS(box.encode(&b)); @@ -1337,7 +1337,7 @@ VOID TEST(KernelMp4Test, SampleDescBox) if (true) { b.skip(-1 * b.pos()); - SrsMp4VisualSampleEntry box; + SrsMp4VisualSampleEntry box = SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); HELPER_EXPECT_SUCCESS(box.decode(&b)); } } @@ -1366,6 +1366,55 @@ VOID TEST(KernelMp4Test, SampleDescBox) } } + if (true) { + char buf[8+8+70]; + SrsBuffer b(buf, sizeof(buf)); + + if (true) { + SrsMp4VisualSampleEntry box = SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); + box.data_reference_index = 1; + EXPECT_EQ((int)sizeof(buf), (int)box.nb_bytes()); + HELPER_EXPECT_SUCCESS(box.encode(&b)); + + stringstream ss; + SrsMp4DumpContext dc; + box.dumps(ss, dc); + + string v = ss.str(); + EXPECT_STREQ("hev1, 86B, refs#1, size=0x0\n", v.c_str()); + } + + if (true) { + b.skip(-1 * b.pos()); + SrsMp4VisualSampleEntry box = SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); + HELPER_EXPECT_SUCCESS(box.decode(&b)); + } + } + + if (true) { + char buf[8]; + SrsBuffer b(buf, sizeof(buf)); + + if (true) { + SrsMp4HvcCBox box; + EXPECT_EQ((int)sizeof(buf), (int)box.nb_bytes()); + HELPER_EXPECT_SUCCESS(box.encode(&b)); + + stringstream ss; + SrsMp4DumpContext dc; + box.dumps(ss, dc); + + string v = ss.str(); + EXPECT_STREQ("hvcC, 8B, HEVC Config: 0B\n \n", v.c_str()); + } + + if (true) { + b.skip(-1 * b.pos()); + SrsMp4HvcCBox box; + HELPER_EXPECT_SUCCESS(box.decode(&b)); + } + } + if (true) { char buf[8+8+20]; SrsBuffer b(buf, sizeof(buf));