From d11a7b2e00792c46dcc4d9a8e4041c4db092e787 Mon Sep 17 00:00:00 2001 From: winlin Date: Fri, 27 Dec 2019 20:47:33 +0800 Subject: [PATCH] For #299, fix some bugs in dash, it works now. 3.0.88 --- README.md | 1 + trunk/src/app/srs_app_dash.cpp | 24 +++-- trunk/src/core/srs_core.hpp | 2 +- trunk/src/kernel/srs_kernel_mp4.cpp | 141 +++++++++++++++++++++++-- trunk/src/kernel/srs_kernel_mp4.hpp | 56 +++++++++- trunk/src/main/srs_main_mp4_parser.cpp | 55 +++------- 6 files changed, 220 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index d79011e3b..f29cae3d3 100755 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ For previous versions, please read: ## V3 changes +* v3.0, 2019-12-27, For [#299][bug #299], fix some bugs in dash, it works now. 3.0.88 * v3.0, 2019-12-27, For [#1544][bug #1544], fix memory leaking for complex error. 3.0.87 * v3.0, 2019-12-27, Add links for flv.js, hls.js and dash.js. * v3.0, 2019-12-26, For [#1105][bug #1105], http server support mp4 range. diff --git a/trunk/src/app/srs_app_dash.cpp b/trunk/src/app/srs_app_dash.cpp index 719c9b110..08296a083 100644 --- a/trunk/src/app/srs_app_dash.cpp +++ b/trunk/src/app/srs_app_dash.cpp @@ -34,6 +34,7 @@ #include #include +#include #include using namespace std; @@ -276,10 +277,17 @@ srs_error_t SrsMpdWriter::get_fragment(bool video, std::string& home, std::strin srs_error_t err = srs_success; home = fragment_home; - - sn = srs_update_system_time() / fragment; + + // We name the segment as advanced N segments, because when we are generating segment at the current time, + // the player may also request the current segment. + srs_assert(fragment); + int64_t number = (srs_update_system_time() / fragment + 1); + // TOOD: FIXME: Should keep the segments continuous, or player may fail. + sn = number; + + // The base time aligned with sn. basetime = sn * fragment; - + if (video) { file_name = "video-" + srs_int2str(sn) + ".m4s"; } else { @@ -292,7 +300,7 @@ srs_error_t SrsMpdWriter::get_fragment(bool video, std::string& home, std::strin SrsDashController::SrsDashController() { req = NULL; - video_tack_id = 2; + video_tack_id = 0; audio_track_id = 1; mpd = new SrsMpdWriter(); vcurrent = acurrent = NULL; @@ -332,6 +340,10 @@ srs_error_t SrsDashController::on_publish() fragment = _srs_config->get_dash_fragment(r->vhost); home = _srs_config->get_dash_path(r->vhost); + if ((err = mpd->on_publish()) != srs_success) { + return srs_error_wrap(err, "mpd"); + } + srs_freep(vcurrent); vcurrent = new SrsFragmentedMp4(); if ((err = vcurrent->initialize(req, true, mpd, video_tack_id)) != srs_success) { @@ -344,10 +356,6 @@ srs_error_t SrsDashController::on_publish() return srs_error_wrap(err, "audio fragment"); } - if ((err = mpd->on_publish()) != srs_success) { - return srs_error_wrap(err, "mpd"); - } - return err; } diff --git a/trunk/src/core/srs_core.hpp b/trunk/src/core/srs_core.hpp index ad8d84e6e..021b75c20 100644 --- a/trunk/src/core/srs_core.hpp +++ b/trunk/src/core/srs_core.hpp @@ -27,7 +27,7 @@ // The version config. #define VERSION_MAJOR 3 #define VERSION_MINOR 0 -#define VERSION_REVISION 87 +#define VERSION_REVISION 88 // The macros generated by configure script. #include diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index 4d7cc3a94..fef9549ea 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -408,6 +408,7 @@ srs_error_t SrsMp4Box::discovery(SrsBuffer* buf, SrsMp4Box** ppbox) case SrsMp4BoxTypeTFHD: box = new SrsMp4TrackFragmentHeaderBox(); break; case SrsMp4BoxTypeTFDT: box = new SrsMp4TrackFragmentDecodeTimeBox(); break; case SrsMp4BoxTypeTRUN: box = new SrsMp4TrackFragmentRunBox(); break; + case SrsMp4BoxTypeSIDX: box = new SrsMp4SegmentIndexBox(); break; // Skip some unknown boxes. case SrsMp4BoxTypeFREE: case SrsMp4BoxTypeSKIP: case SrsMp4BoxTypePASP: box = new SrsMp4FreeSpaceBox(type); break; @@ -4575,6 +4576,120 @@ stringstream& SrsMp4UserDataBox::dumps_detail(stringstream& ss, SrsMp4DumpContex return ss; } +SrsMp4SegmentIndexBox::SrsMp4SegmentIndexBox() +{ + type = SrsMp4BoxTypeSIDX; +} + +SrsMp4SegmentIndexBox::~SrsMp4SegmentIndexBox() +{ +} + +int SrsMp4SegmentIndexBox::nb_header() +{ + return SrsMp4Box::nb_header() + 4+4+4 + (version? 4:8) + 4+4 + 12*entries.size(); +} + +srs_error_t SrsMp4SegmentIndexBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_1bytes(version); + buf->write_3bytes(flags); + buf->write_4bytes(reference_id); + buf->write_4bytes(timescale); + if (!version) { + buf->write_4bytes(earliest_presentation_time); + buf->write_4bytes(first_offset); + } else { + buf->write_8bytes(earliest_presentation_time); + buf->write_8bytes(first_offset); + } + + buf->write_4bytes((uint32_t)entries.size()); + for (int i = 0; i < (int)entries.size(); i++) { + SrsMp4SegmentIndexEntry& entry = entries.at(i); + + uint32_t v = uint32_t(entry.reference_type&0x01)<<31; + v |= entry.referenced_size&0x7fffffff; + buf->write_4bytes(v); + + buf->write_4bytes(entry.subsegment_duration); + + v = uint32_t(entry.starts_with_SAP&0x01)<<31; + v |= uint32_t(entry.SAP_type&0x7)<<28; + v |= entry.SAP_delta_time&0xfffffff; + buf->write_4bytes(v); + } + + return err; +} + +srs_error_t SrsMp4SegmentIndexBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + version = buf->read_1bytes(); + flags = buf->read_3bytes(); + reference_id = buf->read_4bytes(); + timescale = buf->read_4bytes(); + if (!version) { + earliest_presentation_time = buf->read_4bytes(); + first_offset = buf->read_4bytes(); + } else { + earliest_presentation_time = buf->read_8bytes(); + first_offset = buf->read_8bytes(); + } + + uint32_t nn_entries = (uint32_t)(buf->read_4bytes() & 0xffff); + for (uint32_t i = 0; i < nn_entries; i++) { + SrsMp4SegmentIndexEntry entry; + + uint32_t v = buf->read_4bytes(); + entry.reference_type = uint8_t((v&0x80000000)>>31); + entry.referenced_size = v&0x7fffffff; + + entry.subsegment_duration = buf->read_4bytes(); + + v = buf->read_4bytes(); + entry.starts_with_SAP = uint8_t((v&0x80000000)>>31); + entry.SAP_type = uint8_t((v&0x70000000)>>28); + entry.SAP_delta_time = v&0xfffffff; + + entries.push_back(entry); + } + + return err; +} + +stringstream& SrsMp4SegmentIndexBox::dumps_detail(stringstream& ss, SrsMp4DumpContext dc) +{ + SrsMp4Box::dumps_detail(ss, dc); + + ss << ", v" << (int)version << ", flags=" << flags << ", refs#" << reference_id + << ", TBN=" << timescale << ", ePTS=" << earliest_presentation_time; + + for (int i = 0; i < (int)entries.size(); i++) { + SrsMp4SegmentIndexEntry& entry = entries.at(i); + + ss << endl; + srs_padding(ss, dc.indent()); + ss << "#" << i << ", ref=" << (int)entry.reference_type << "/" << entry.referenced_size + << ", duration=" << entry.subsegment_duration << ", SAP=" << (int)entry.starts_with_SAP + << "/" << (int)entry.SAP_type << "/" << entry.SAP_delta_time; + } + + return ss; +} + SrsMp4Sample::SrsMp4Sample() { type = SrsFrameTypeForbidden; @@ -5930,8 +6045,8 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) SrsAutoFree(SrsMp4FileTypeBox, ftyp); if (true) { ftyp->major_brand = SrsMp4BoxBrandISO5; - ftyp->minor_version = 0; - ftyp->set_compatible_brands(SrsMp4BoxBrandISOM, SrsMp4BoxBrandISO5, SrsMp4BoxBrandDASH, SrsMp4BoxBrandMP42); + ftyp->minor_version = 512; + ftyp->set_compatible_brands(SrsMp4BoxBrandISO6, SrsMp4BoxBrandMP41); } // Write moov. @@ -6135,7 +6250,7 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) uint8_t* data = new uint8_t[nb_data]; SrsAutoFreeA(uint8_t, data); - SrsBuffer* buffer = new SrsBuffer(); + SrsBuffer* buffer = new SrsBuffer((char*)data, nb_data); SrsAutoFree(SrsBuffer, buffer); if ((err = ftyp->encode(buffer)) != srs_success) { @@ -6160,7 +6275,6 @@ SrsMp4M2tsSegmentEncoder::SrsMp4M2tsSegmentEncoder() buffer = new SrsBuffer(); sequence_number = 0; decode_basetime = 0; - data_offset = 0; mdat_bytes = 0; } @@ -6186,7 +6300,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::initialize(ISrsWriter* w, uint32_t sequenc styp->major_brand = SrsMp4BoxBrandMSDH; styp->minor_version = 0; - styp->set_compatible_brands(SrsMp4BoxBrandMSDH, SrsMp4BoxBrandDASH); + styp->set_compatible_brands(SrsMp4BoxBrandMSDH, SrsMp4BoxBrandMSIX); int nb_data = styp->nb_bytes(); std::vector data(nb_data); @@ -6202,9 +6316,8 @@ srs_error_t SrsMp4M2tsSegmentEncoder::initialize(ISrsWriter* w, uint32_t sequenc if ((err = writer->write(&data[0], nb_data, NULL)) != srs_success) { return srs_error_wrap(err, "write styp"); } - - data_offset = nb_data; } + return err; } @@ -6230,7 +6343,11 @@ srs_error_t SrsMp4M2tsSegmentEncoder::write_sample(SrsMp4HandlerType ht, ps->tbn = 1000; ps->dts = dts; ps->pts = pts; - ps->data = sample; + + // We should copy the sample data, which is shared ptr from video/audio message. + // Furthermore, we do free the data when freeing the sample. + ps->data = new uint8_t[nb_sample]; + memcpy(ps->data, sample, nb_sample); ps->nb_data = nb_sample; // Append to manager to build the moof. @@ -6254,7 +6371,10 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) // and we will update its header(size) when flush. SrsMp4MediaDataBox* mdat = new SrsMp4MediaDataBox(); SrsAutoFree(SrsMp4MediaDataBox, mdat); - + + // Although the sidx is not required to start play DASH, but it's required for AV sync. + // TODO: FIXME: Insert a sidx box. + // Write moof. if (true) { SrsMp4MovieFragmentBox* moof = new SrsMp4MovieFragmentBox(); @@ -6288,7 +6408,8 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) } int nb_data = moof->nb_bytes(); - trun->data_offset = (int32_t)(data_offset + nb_data + mdat->sz_header()); + // @remark Remember the data_offset of turn is size(moof)+header(mdat), not including styp or sidx. + trun->data_offset = (int32_t)(nb_data + mdat->sz_header()); uint8_t* data = new uint8_t[nb_data]; SrsAutoFreeA(uint8_t, data); diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index 67608673c..c51e30ad0 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -123,6 +123,7 @@ enum SrsMp4BoxType SrsMp4BoxTypeTFHD = 0x74666864, // 'tfhd' SrsMp4BoxTypeTFDT = 0x74666474, // 'tfdt' SrsMp4BoxTypeTRUN = 0x7472756e, // 'trun' + SrsMp4BoxTypeSIDX = 0x73696478, // 'sidx' }; // 8.4.3.3 Semantics @@ -145,9 +146,11 @@ enum SrsMp4BoxBrand SrsMp4BoxBrandAVC1 = 0x61766331, // 'avc1' SrsMp4BoxBrandMP41 = 0x6d703431, // 'mp41' SrsMp4BoxBrandISO5 = 0x69736f35, // 'iso5' + SrsMp4BoxBrandISO6 = 0x69736f36, // 'iso6' SrsMp4BoxBrandMP42 = 0x6d703432, // 'mp42' SrsMp4BoxBrandDASH = 0x64617368, // 'dash' SrsMp4BoxBrandMSDH = 0x6d736468, // 'msdh' + SrsMp4BoxBrandMSIX = 0x6d736978, // 'msix' }; // The context to dump. @@ -179,8 +182,10 @@ public: // An extended type; in this case, the type field is set to ‘uuid’. SrsMp4BoxType type; // For box 'uuid'. + // TODO: FIXME: Should double check buffer. std::vector usertype; protected: + // TODO: FIXME: Should double check buffer. std::vector boxes; private: // The position at buffer to start demux the box. @@ -274,6 +279,7 @@ public: uint32_t minor_version; private: // A list, to the end of the box, of brands + // TODO: FIXME: Should double check buffer. std::vector compatible_brands; public: SrsMp4FileTypeBox(); @@ -506,6 +512,7 @@ public: uint32_t first_sample_flags; // all fields in the following array are optional public: + // TODO: FIXME: Should double check buffer. std::vector entries; public: SrsMp4TrackFragmentRunBox(); @@ -595,6 +602,7 @@ public: class SrsMp4FreeSpaceBox : public SrsMp4Box { private: + // TODO: FIXME: Should double check buffer. std::vector data; public: SrsMp4FreeSpaceBox(SrsMp4BoxType v); @@ -902,6 +910,7 @@ class SrsMp4EditListBox : public SrsMp4FullBox { public: // An integer that gives the number of entries in the following table + // TODO: FIXME: Should double check buffer. std::vector entries; public: SrsMp4EditListBox(); @@ -1152,6 +1161,7 @@ public: class SrsMp4DataReferenceBox : public SrsMp4FullBox { private: + // TODO: FIXME: Should double check buffer. std::vector entries; public: SrsMp4DataReferenceBox(); @@ -1273,6 +1283,7 @@ public: class SrsMp4AvccBox : public SrsMp4Box { public: + // TODO: FIXME: Should double check buffer. std::vector avc_config; public: SrsMp4AvccBox(); @@ -1383,6 +1394,7 @@ class SrsMp4DecoderSpecificInfo : public SrsMp4BaseDescriptor public: // AAC Audio Specific Config. // 1.6.2.1 AudioSpecificConfig, in ISO_IEC_14496-3-AAC-2001.pdf, page 33. + // TODO: FIXME: Should double check buffer. std::vector asc; public: SrsMp4DecoderSpecificInfo(); @@ -1449,6 +1461,7 @@ public: // if (streamDependenceFlag) uint16_t dependsOn_ES_ID; // if (URL_Flag) + // TODO: FIXME: Should double check buffer. std::vector URLstring; // if (OCRstreamFlag) uint16_t OCR_ES_Id; @@ -1494,6 +1507,7 @@ public: class SrsMp4SampleDescriptionBox : public SrsMp4FullBox { private: + // TODO: FIXME: Should double check buffer. std::vector entries; public: SrsMp4SampleDescriptionBox(); @@ -1543,6 +1557,7 @@ class SrsMp4DecodingTime2SampleBox : public SrsMp4FullBox { public: // An integer that gives the number of entries in the following table. + // TODO: FIXME: Should double check buffer. std::vector entries; private: // The index for counter to calc the dts for samples. @@ -1783,6 +1798,7 @@ public: class SrsMp4UserDataBox : public SrsMp4Box { public: + // TODO: FIXME: Should double check buffer. std::vector data; public: SrsMp4UserDataBox(); @@ -1795,6 +1811,44 @@ public: virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); }; +// The entry for SegmentIndexBox(sidx) for MPEG-DASH. +// @doc https://patches.videolan.org/patch/103/ +struct SrsMp4SegmentIndexEntry +{ + uint8_t reference_type; // 1bit + uint32_t referenced_size; // 31bits + uint32_t subsegment_duration; // 32bits + uint8_t starts_with_SAP; // 1bit + uint8_t SAP_type; // 3bits + uint32_t SAP_delta_time; // 28bits +}; + +// The SegmentIndexBox(sidx) for MPEG-DASH. +// @doc https://gpac.wp.imt.fr/2012/02/01/dash-support/ +// @doc https://patches.videolan.org/patch/103/ +// @doc https://github.com/necccc/iso-bmff-parser-stream/blob/master/lib/box/sidx.js +class SrsMp4SegmentIndexBox : public SrsMp4Box +{ +public: + uint8_t version; + uint32_t flags; + uint32_t reference_id; + uint32_t timescale; + uint64_t earliest_presentation_time; + uint32_t first_offset; + // TODO: FIXME: Should double check buffer. + std::vector entries; +public: + SrsMp4SegmentIndexBox(); + virtual ~SrsMp4SegmentIndexBox(); +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + // Generally, a MP4 sample contains a frame, for example, a video frame or audio frame. class SrsMp4Sample { @@ -2063,8 +2117,6 @@ private: uint32_t nb_videos; uint64_t mdat_bytes; SrsMp4SampleManager* samples; -private: - uint64_t data_offset; public: SrsMp4M2tsSegmentEncoder(); virtual ~SrsMp4M2tsSegmentEncoder(); diff --git a/trunk/src/main/srs_main_mp4_parser.cpp b/trunk/src/main/srs_main_mp4_parser.cpp index 86633d4f3..8518e10d0 100644 --- a/trunk/src/main/srs_main_mp4_parser.cpp +++ b/trunk/src/main/srs_main_mp4_parser.cpp @@ -40,28 +40,19 @@ using namespace std; ISrsLog* _srs_log = new SrsConsoleLog(SrsLogLevelTrace, false); ISrsThreadContext* _srs_context = new SrsThreadContext(); -int parse(std::string mp4_file, bool verbose) +srs_error_t parse(std::string mp4_file, bool verbose) { - int ret = ERROR_SUCCESS; srs_error_t err = srs_success; SrsFileReader fr; if ((err = fr.open(mp4_file)) != srs_success) { - // TODO: FIXME: Use error - ret = srs_error_code(err); - srs_freep(err); - srs_error("Open MP4 file failed, ret=%d", ret); - return ret; + return srs_error_wrap(err, "open mp4 file %s", mp4_file.c_str()); } srs_trace("MP4 file open success"); SrsMp4BoxReader br; if ((err = br.initialize(&fr)) != srs_success) { - // TODO: FIXME: Use error - ret = srs_error_code(err); - srs_freep(err); - srs_error("Open MP4 box reader failed, ret=%d", ret); - return ret; + return srs_error_wrap(err, "open box reader"); } srs_trace("MP4 box reader open success"); @@ -74,34 +65,21 @@ int parse(std::string mp4_file, bool verbose) SrsAutoFree(SrsMp4Box, box); if ((err = br.read(stream, &box)) != srs_success) { - // TODO: FIXME: Use error - ret = srs_error_code(err); - srs_freep(err); - if (ret != ERROR_SYSTEM_FILE_EOF) { - srs_error("Read MP4 box failed, ret=%d", ret); - } else { + if (srs_error_code(err) == ERROR_SYSTEM_FILE_EOF) { fprintf(stderr, "\n"); } - return ret; + return srs_error_wrap(err, "read box"); } SrsBuffer* buffer = new SrsBuffer(stream->bytes(), stream->length()); SrsAutoFree(SrsBuffer, buffer); if ((err = box->decode(buffer)) != srs_success) { - // TODO: FIXME: Use error - ret = srs_error_code(err); - srs_freep(err); - srs_error("Decode the box failed, ret=%d", ret); - return ret; + return srs_error_wrap(err, "decode box"); } if ((err = br.skip(box, stream)) != srs_success) { - // TODO: FIXME: Use error - ret = srs_error_code(err); - srs_freep(err); - srs_error("Skip MP4 box failed, ret=%d", ret); - return ret; + return srs_error_wrap(err, "skip box"); } SrsMp4DumpContext ctx; @@ -112,13 +90,11 @@ int parse(std::string mp4_file, bool verbose) fprintf(stderr, "%s", box->dumps(ss, ctx).str().c_str()); } - return ret; + return err; } int main(int argc, char** argv) { - int ret = ERROR_SUCCESS; - printf("SRS MP4 parser/%d.%d.%d, parse and show the mp4 boxes structure.\n", VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION); @@ -140,13 +116,16 @@ int main(int argc, char** argv) } srs_trace("Parse MP4 file %s, verbose=%d", mp4_file.c_str(), verbose); - ret = parse(mp4_file, verbose); - - if (ret == ERROR_SYSTEM_FILE_EOF) { + srs_error_t err = parse(mp4_file, verbose); + int code = srs_error_code(err); + + if (code == ERROR_SYSTEM_FILE_EOF) { srs_trace("Parse complete"); - return 0; + } else { + srs_error("Parse error %s", srs_error_desc(err).c_str()); } - - return ret; + + srs_freep(err); + return code; }